Locked History Actions

Diff for "FLISOL2018/DRBD"

Differences between revisions 1 and 6 (spanning 5 versions)
⇤ ← Revision 1 as of 2018-05-01 15:38:43 →
Size: 39793
Editor: Michael Roessler
Comment:
← Revision 6 as of 2018-05-01 16:43:14 → ⇥
Size: 40318
Editor: Michael Roessler
Comment:
Deletions are marked like this. Additions are marked like this.
Line 27: Line 27:
* 3 Raspberry PI 3 B, eth0 @ 100 Mbit/s
* 2 USB-Sticks 64 GByte (1 USB-Stick de 64 GB para backup)
* 8-Port Switch 1 GBit/s
* Fuente de energía: Raspberry original, Samsung, USB-Hub Activo		  * 3 Raspberry PI 3 B, eth0 @ 100 Mbit/s
 * 2 USB-Sticks 64 GByte (1 USB-Stick de 64 GB para backup)
 * 8-Port Switch 1 GBit/s
 * Fuente de energía: Raspberry original, Samsung, USB-Hub Activo
Line 37: Line 37:
{{{
Line 42: Line 42:
}}}
Line 44: Line 44:
{{{
Line 47: Line 47:
}}}
Line 49: Line 49:
{{{
Line 51: Line 51:
}}}
Line 55: Line 55:
{{{
Line 57: Line 57:
}}}
Line 63: Line 63:

  [Global]
  Comment "Global section"
  NodeBase "ha-iscsi2.203.istgt"
 
  PidFile /var/run/istgt.pid
  AuthFile /etc/istgt/auth.conf
 
  MediaDirectory /var/istgt
 
  LogFacility "local7"
 
  Timeout 30
  NopInInterval 20
 
  DiscoveryAuthMethod Auto
 
  MaxSessions 16
  MaxConnections 4
 
  MaxR2T 32
 
  MaxOutstandingR2T 16
  DefaultTime2Wait 2
  DefaultTime2Retain 60
  FirstBurstLength 262144
  MaxBurstLength 1048576
  MaxRecvDataSegmentLength 262144
 
  InitialR2T Yes
  ImmediateData Yes
  DataPDUInOrder Yes
  DataSequenceInOrder Yes
  ErrorRecoveryLevel 0
 
 [UnitControl]		 {{{
[Global]
 Comment "Global section"
 NodeBase "ha-iscsi2.203.istgt"
 
 PidFile /var/run/istgt.pid
 AuthFile /etc/istgt/auth.conf
 
 MediaDirectory /var/istgt
 
 LogFacility "local7"
 
 Timeout 30
 NopInInterval 20
 
 DiscoveryAuthMethod Auto
 
 MaxSessions 16
 MaxConnections 4
 
 MaxR2T 32
 
 MaxOutstandingR2T 16
 DefaultTime2Wait 2
 DefaultTime2Retain 60
 FirstBurstLength 262144
 MaxBurstLength 1048576
 MaxRecvDataSegmentLength 262144
 
 InitialR2T Yes
 ImmediateData Yes
 DataPDUInOrder Yes
 DataSequenceInOrder Yes
 ErrorRecoveryLevel 0
 
[UnitControl]
Line 105: Line 105:
 [PortalGroup1] [PortalGroup1]
Line 109: Line 109:
 [InitiatorGroup1] [InitiatorGroup1]
Line 114: Line 114:
 [LogicalUnit1] [LogicalUnit1]
Line 125: Line 125:
}}}
Line 128: Line 128:
{{{
Line 133: Line 134:
}}}
Line 135: Line 136:

 [AuthGroup1]		 {{{
[AuthGroup1]
Line 140: Line 141:
 [AuthGroup9999] [AuthGroup9999]
Line 143: Line 144:
 [AuthGroup10000] [AuthGroup10000]
Line 147: Line 148:
}}}
Line 149: Line 151:
{{{
Line 151: Line 153:
}}}
Line 155: Line 157:

 global {
  dialog-refresh 1;
  minor-count 5;
  usage-count no;		 {{{
global {
 dialog-refresh 1;
 minor-count 5;
 usage-count no;
}
common {
}
resource r0 {
 protocol C;
 handlers {
  split-brain "/usr/lib/drbd/notify-split-brain.sh root";
Line 161: Line 170:
 common {  disk {
    on-io-error pass_on;
Line 163: Line 173:
 resource r0 {
  protocol C;
  handlers {
   split-brain "/usr/lib/drbd/notify-split-brain.sh root";		  syncer {
 verify-alg sha1;
 rate 12M;
Line 168: Line 177:
  disk {
     on-io-error pass_on;
  }
  syncer {
  verify-alg sha1;
  rate 12M;
  }
  net {
       allow-two-primaries;
  after-sb-0pri discard-younger-primary;
    after-sb-1pri call-pri-lost-after-sb;
    after-sb-2pri call-pri-lost-after-sb;
  }
  startup {
       become-primary-on both;
  }
  on ha-iscsi1 {
     device /dev/drbd0;
     address 10.250.30.202:7788;
     meta-disk /dev/sda1[0];
     disk /dev/sda2;
  }
  on ha-iscsi2 {
     device /dev/drbd0;
     address 10.250.30.203:7788;
     meta-disk /dev/sda1[0];
     disk /dev/sda2;
  }		  net {
   allow-two-primaries;
   after-sb-0pri discard-younger-primary;
   after-sb-1pri call-pri-lost-after-sb;
   after-sb-2pri call-pri-lost-after-sb;
Line 197: Line 183:
 startup {
      become-primary-on both;
 }
 on ha-iscsi1 {
    device /dev/drbd0;
    address 10.250.30.202:7788;
    meta-disk /dev/sda1[0];
    disk /dev/sda2;
 }
 on ha-iscsi2 {
    device /dev/drbd0;
    address 10.250.30.203:7788;
    meta-disk /dev/sda1[0];
    disk /dev/sda2;
 }
}
}}}
Line 201: Line 203:
{{{
Line 203: Line 205:
}}}
Line 205: Line 207:
{{{
Line 207: Line 209:
}}}
Line 209: Line 211:
{{{
Line 211: Line 213:
}}}
Line 213: Line 215:
{{{
Line 215: Line 217:
}}}
Line 217: Line 219:
{{{
Line 219: Line 221:
}}}
Line 223: Line 225:
{{{
Line 225: Line 227:
}}}
Line 229: Line 231:

 totem {		 {{{
totem {
Line 249: Line 251:
 logging { logging {
Line 265: Line 267:
 quorum { quorum {
Line 269: Line 271:
 nodelist { nodelist {
Line 279: Line 281:
{{{
Line 281: Line 283:
 ^stop^start
  systemctl start corosync
}}}
Line 284: Line 286:
{{{
Line 286: Line 288:
 ^stop^start
  systemctl start pacemaker
}}}
Line 293: Line 295:
{{{
Line 295: Line 297:
}}}
Line 297: Line 299:
{{{
Line 300: Line 302:
}}}
Line 302: Line 304:
{{{
Line 305: Line 307:

Salide de "crm status"		 }}}
Salida de "crm status"
{{{
Line 322: Line 324:
}}}
Line 324: Line 326:
{{{
Line 327: Line 329:
}}}
Line 329: Line 331:
{{{
Line 334: Line 336:
}}}
Line 336: Line 338:
{{{
Line 338: Line 340:
}}}
Line 340: Line 342:
* scsi_dh_alua
* scsi_dh_emc
* scsi_dh_rdac
* dm-multipath		  * scsi_dh_alua
 * scsi_dh_emc
 * scsi_dh_rdac
 * dm-multipath
{{{
Line 348: Line 350:
}}}
{{{
Line 352: Line 355:
}}}
Line 354: Line 357:
{{{
Line 363: Line 366:
}}}
Line 365: Line 368:
{{{
Line 367: Line 370:
}}}
Line 369: Line 372:
{{{
Line 371: Line 374:
}}}
Line 373: Line 376:
{{{
Line 378: Line 381:
}}}
Line 380: Line 383:
{{{
Line 382: Line 385:
}}}
Line 388: Line 391:
{{{
Line 394: Line 397:
}}}
Line 398: Line 401:
{{{
Line 401: Line 404:
}}}
Line 403: Line 407:

 iscsid.startup = /usr/sbin/iscsid
 node.startup = automatic
 node.startup = manual
 node.leading_login = No
 node.session.timeo.replacement_timeout = 120
 node.conn[0].timeo.login_timeout = 15
 node.conn[0].timeo.logout_timeout = 15
 node.conn[0].timeo.noop_out_interval = 5
 node.conn[0].timeo.noop_out_timeout = 5
 node.session.err_timeo.abort_timeout = 15
 node.session.err_timeo.lu_reset_timeout = 30
 node.session.err_timeo.tgt_reset_timeout = 30
 node.session.initial_login_retry_max = 8
 node.session.cmds_max = 128
 node.session.queue_depth = 32
 node.session.xmit_thread_priority = -20
 node.session.iscsi.InitialR2T = No
 node.session.iscsi.ImmediateData = Yes
 node.session.iscsi.FirstBurstLength = 262144
 node.session.iscsi.MaxBurstLength = 16776192		 {{{
iscsid.startup = /usr/sbin/iscsid
node.startup = automatic
node.startup = manual
node.leading_login = No
node.session.timeo.replacement_timeout = 120
node.conn[0].timeo.login_timeout = 15
node.conn[0].timeo.logout_timeout = 15
node.conn[0].timeo.noop_out_interval = 5
node.conn[0].timeo.noop_out_timeout = 5
node.session.err_timeo.abort_timeout = 15
node.session.err_timeo.lu_reset_timeout = 30
node.session.err_timeo.tgt_reset_timeout = 30
node.session.initial_login_retry_max = 8
node.session.cmds_max = 128
node.session.queue_depth = 32
node.session.xmit_thread_priority = -20
node.session.iscsi.InitialR2T = No
node.session.iscsi.ImmediateData = Yes
node.session.iscsi.FirstBurstLength = 262144
node.session.iscsi.MaxBurstLength = 16776192
}}}
Line 428: Line 432:
{{{
Line 432: Line 436:
}}}
Line 434: Line 438:
{{{
Line 438: Line 442:
}}}
Line 440: Line 444:
{{{
Line 445: Line 449:
}}}
Line 449: Line 453:

 blacklist {
 devnode "ram"
 devnode "mmcblk0"
 devnode "sdc"
 }
 
 defaults {
 user_friendly_names yes
 path_selector "round-robin 0"
 }		 {{{
blacklist {
devnode "ram"
devnode "mmcblk0"
devnode "sdc"
}
 
defaults {
user_friendly_names yes
path_selector "round-robin 0"
}
}}}
Line 462: Line 466:
{{{
Line 464: Line 468:
 ^stop^start
  systemctl start multipath-tools
}}}
Line 467: Line 471:
{{{
Line 473: Line 477:
}}}
Line 475: Line 479:
{{{
Line 478: Line 482:
}}}
Line 480: Line 484:
{{{
Line 482: Line 486:
}}}
Line 484: Line 488:
{{{
Line 486: Line 490:

Si se crea un nuevo container con ese commando ...		 }}}
Se crea un nuevo container con ese commando ...
{{{
Line 490: Line 494:
}}}
Line 494: Line 498:
{{{
Line 496: Line 500:
}}}
Line 498: Line 502:

 #lxc.include = /usr/share/lxc/config/nesting.conf
 # (Be aware this has security implications)
 
 lxc.network.type = veth
 lxc.network.flags = up
 lxc.network.link = br0
 lxc.network.ipv4 = 10.250.30.206/24
 lxc.network.ipv4.gateway = 10.250.30.4
 lxc.rootfs = /dev/cvg1/wiki
 lxc.rootfs.backend = lvm
 
 # Common configuration
 lxc.include = /usr/share/lxc/config/debian.common.conf
 
 # Container specific configuration
 lxc.tty = 4
 lxc.utsname = wiki
 lxc.arch = armhf
 {{{
#lxc.include = /usr/share/lxc/config/nesting.conf
# (Be aware this has security implications)
 
lxc.network.type = veth
lxc.network.flags = up
lxc.network.link = br0
lxc.network.ipv4 = 10.250.30.206/24
lxc.network.ipv4.gateway = 10.250.30.4
lxc.rootfs = /dev/cvg1/wiki
lxc.rootfs.backend = lvm
 
# Common configuration
lxc.include = /usr/share/lxc/config/debian.common.conf
 
# Container specific configuration
lxc.tty = 4
lxc.utsname = wiki
lxc.arch = armhf
}}}
Line 519: Line 523:
{{{
Line 521: Line 525:
}}}
Line 527: Line 531:
{{{
Line 529: Line 533:
 # prevent editing and reading by anons (except for exception listed below):
 $wgGroupPermissions['*']['edit'] = false;
 $wgGroupPermissions['*']['createaccount'] = false;
 $wgGroupPermissions['*']['read'] = true;
 
 # same for normal registered users:
 $wgGroupPermissions['user']['edit'] = false;
 $wgGroupPermissions['user']['read'] = true;
 
 # allow everyone read access to these pages:
 #$wgWhitelistRead = array( "Main Page", "Public stuff" );
 
 # allow sysops to read and edit normally:
 $wgGroupPermissions['sysop']['edit'] = true;
 $wgGroupPermissions['sysop']['read'] = true;
 # prevent editing and reading by anons (except for exception listed below):
$wgGroupPermissions['*']['edit'] = false;
$wgGroupPermissions['*']['createaccount'] = false;
$wgGroupPermissions['*']['read'] = true;
 
# same for normal registered users:
$wgGroupPermissions['user']['edit'] = false;
$wgGroupPermissions['user']['read'] = true;
 
# allow everyone read access to these pages:
#$wgWhitelistRead = array( "Main Page", "Public stuff" );
 
# allow sysops to read and edit normally:
$wgGroupPermissions['sysop']['edit'] = true;
$wgGroupPermissions['sysop']['read'] = true;
}}}
Line 546: Line 550:
{{{
...
wfLoadExtension( 'VisualEditor' );
 
// Enable by default for everybody
$wgDefaultUserOptions['visualeditor-enable'] = 1;
// Optional: Set VisualEditor as the default for anonymous users
// otherwise they will have to switch to VE
$wgDefaultUserOptions['visualeditor-editor'] = "visualeditor";
 
// Don't allow users to disable it
$wgHiddenPrefs[] = 'visualeditor-enable';
....
}}}
Activar cargar
{{{
...
$wgEnableUploads = true;
...
}}}
===== Configuración Apache2 en Container wiki =====

Cambio este archivo:
{{{
 /etc/apache2/sites-enabled/000-default.conf
}}}
Explicaciónes están en los commentarios
{{{
Line 548: Line 579:
 wfLoadExtension( 'VisualEditor' );
 
 // Enable by default for everybody
 $wgDefaultUserOptions['visualeditor-enable'] = 1;
 // Optional: Set VisualEditor as the default for anonymous users
 // otherwise they will have to switch to VE
 $wgDefaultUserOptions['visualeditor-editor'] = "visualeditor";
 
 // Don't allow users to disable it
 $wgHiddenPrefs[] = 'visualeditor-enable';
 ....

Activar cargar
 
 ...
 $wgEnableUploads = true;
 ...

===== Configuración Apache2 en Container wiki =====

Cambio este archivo:

 /etc/apache2/sites-enabled/000-default.conf

Explicaciónes están en los commentarios

 ...
 <Virtualhost ...		 <Virtualhost ...
Line 597: Line 601:
 </Virtualhost> </Virtualhost>
}}}
Line 604: Line 608:
{{{
Line 614: Line 618:
}}}
Line 616: Line 620:
{{{
Line 618: Line 622:
}}}
Line 629: Line 633:
* lxc1pi
* ha-iscsi1
* ha-iscsi2		  * lxc1pi
 * ha-iscsi1
 * ha-iscsi2
Line 636: Line 640:
{{{
Line 638: Line 642:
}}}

Se prueba con ...
{{{
 lxc-ls --fancy
}}}
Line 640: Line 649:
{{{
Line 642: Line 651:

Se prueba con ...

 lxc-ls --fancy
 }}}
Line 649: Line 654:
* Apagar istgt  * Apagar istgt
{{{
Line 652: Line 657:

* Apagar drbd		 }}}
 * Apagar drbd
{{{
Line 656: Line 661:

Apagar sistema		 }}}
* Apagar sistema
{{{
Line 660: Line 665:
}}}
Line 665: Line 670:
* ha-iscsi1 y ha-iscsi2
* lxc1pi		  * ha-iscsi1 y ha-iscsi2
 * lxc1pi
Line 670: Line 675:
* Iniciar drbd  * Iniciar drbd
{{{
Line 673: Line 678:

* iniciar istgt		 }}}
 * iniciar istgt
{{{
Line 677: Line 682:
}}}
Line 679: Line 684:

 crm status ...		 {{{
 crm status
}}}
Line 683: Line 688:
{{{
Line 685: Line 690:
}}}
Line 687: Line 692:
{{{
Line 689: Line 694:
}}}
Line 695: Line 700:
{{{
Line 697: Line 702:
}}}
Line 699: Line 704:
{{{
Line 701: Line 706:
}}}
Line 703: Line 708:
{{{
Line 705: Line 710:
}}}
Line 707: Line 712:
{{{
Line 709: Line 714:
}}}
Line 717: Line 722:
{{{
Line 720: Line 725:
}}}
Line 722: Line 727:
{{{
Line 724: Line 729:
}}}
Line 726: Line 731:
{{{
Line 729: Line 734:
}}}
Line 733: Line 738:
{{{
Line 736: Line 741:
}}}
Line 738: Line 743:
{{{
Line 740: Line 745:
}}}
Line 744: Line 749:
{{{
Line 746: Line 751:
}}}
Line 748: Line 753:
{{{
Line 750: Line 755:
}}}
Line 752: Line 757:
{{{
Line 754: Line 759:
}}}
Line 758: Line 763:
{{{
Line 760: Line 765:
}}}
Line 762: Line 767:
{{{
Line 765: Line 770:
}}}
Line 775: Line 780:
{{{
Line 779: Line 784:
}}}
Line 785: Line 790:
{{{
Line 790: Line 795:

'''Se puede usar estes comandos aunque no hay errores.'''		 }}}
'''Se puede usar estes comandos aunque no haya errores.'''
Line 800: Line 805:
{{{
Line 804: Line 809:
}}}
Line 806: Line 811:
{{{
Line 809: Line 814:
}}}
Line 811: Line 816:
{{{
Line 813: Line 818:
}}}
Line 815: Line 820:
{{{
Line 817: Line 822:
}}}
Line 819: Line 824:
{{{
Line 821: Line 826:
}}}
Line 823: Line 828:
{{{
Line 858: Line 863:
}}}
Line 860: Line 865:
{{{
Line 862: Line 867:
}}}
Line 864: Line 869:
{{{
Line 899: Line 904:
}}}
Line 903: Line 908:
{{{
Line 905: Line 910:
}}}
Line 907: Line 912:
{{{
Line 909: Line 914:
}}}
Line 911: Line 916:
{{{
Line 913: Line 918:
}}}
Line 915: Line 920:
{{{
Line 917: Line 922:

===USB-Stick solo: Probar potencia de disco duro: Datos random, escribiendo ===		 }}}
=== USB-Stick solo: Probar potencia de disco duro: Datos random, escribiendo ===
{{{
Line 921: Line 926:
}}}
Line 923: Line 928:
{{{
Line 958: Line 963:
}}}
Line 960: Line 965:
{{{
Line 962: Line 967:
}}}
Line 964: Line 969:
{{{
Line 998: Line 1003:

"apt upgrade" falló con linux-firmware -> solución: otra vez "apt upgrade"		 }}}
Line 1009: Line 1013:
* DRBD-Nodes: Muchos IO-Errors en sda. Los Observé con "dmesg"
* LXC-guest: Rootfs en Ext4 está read only.
* DRBD-Node: uno es "inconsistent". Lo observé con "cat /proc/drbd". "UpToDate" es normal.		  * DRBD-Nodes: Muchos IO-Errors en sda. Los Observé con "dmesg"
 * LXC-guest: Rootfs en Ext4 está read only.
 * DRBD-Node: uno es "inconsistent". Lo observé con "cat /proc/drbd". "UpToDate" es normal.
Line 1014: Line 1018:
{{{
Line 1019: Line 1023:
}}}
Line 1024: Line 1028:
Otra causo posible: No hay suficiente Volt. Otra causa posible: No hay suficiente Volt.
Line 1030: Line 1034:
 \#!/bin/sh
 DEV=eth0
 IPT=/sbin/iptables
 TC=/sbin/tc
 $IPT -t mangle -F
 $TC qdisc del dev $DEV ingress > /dev/null 2>&1
 $TC qdisc del dev $DEV root > /dev/null 2>&1
 $TC qdisc del dev lo root > /dev/null 2>&1
 $TC qdisc add dev $DEV root handle 1:0 htb default 12 r2q 6
 $TC class add dev $DEV parent 1:0 classid 1:1 htb rate 100mbit ceil 100mbit
 $TC class add dev $DEV parent 1:1 classid 1:10 htb rate 10mbit ceil 100mbit prio 0
 $TC class add dev $DEV parent 1:1 classid 1:11 htb rate 50mbit ceil 100mbit prio 1
 $TC class add dev $DEV parent 1:1 classid 1:12 htb rate 40mbit ceil 100mbit prio 2
 $IPT -A POSTROUTING -t mangle -o $DEV -p tcp -m length --length :64 -j MARK --set-mark 10
 $IPT -A POSTROUTING -t mangle -o $DEV -p udp --dport 5405 -j MARK --set-mark 10
 $IPT -A POSTROUTING -t mangle -o $DEV -p udp --sport 5405 -j MARK --set-mark 10
 $IPT -A POSTROUTING -t mangle -o $DEV -p tcp --dport 7788 -j MARK --set-mark 11
 $IPT -A POSTROUTING -t mangle -o $DEV -p tcp --sport 7788 -j MARK --set-mark 11
 $TC filter add dev $DEV parent 1:0 prio 0 protocol ip handle 10 fw flowid 1:10
 $TC filter add dev $DEV parent 1:0 prio 0 protocol ip handle 11 fw flowid 1:11
 $TC qdisc add dev $DEV parent 1:10 handle 10: sfq perturb 10
 $TC qdisc add dev $DEV parent 1:11 handle 11: sfq perturb 10
 $TC qdisc add dev $DEV parent 1:12 handle 12: sfq perturb 10		 {{{
#!/bin/sh
DEV=eth0
IPT=/sbin/iptables
TC=/sbin/tc
$IPT -t mangle -F
$TC qdisc del dev $DEV ingress > /dev/null 2>&1
$TC qdisc del dev $DEV root > /dev/null 2>&1
$TC qdisc del dev lo root > /dev/null 2>&1
$TC qdisc add dev $DEV root handle 1:0 htb default 12 r2q 6
$TC class add dev $DEV parent 1:0 classid 1:1 htb rate 100mbit ceil 100mbit
$TC class add dev $DEV parent 1:1 classid 1:10 htb rate 10mbit ceil 100mbit prio 0
$TC class add dev $DEV parent 1:1 classid 1:11 htb rate 50mbit ceil 100mbit prio 1
$TC class add dev $DEV parent 1:1 classid 1:12 htb rate 40mbit ceil 100mbit prio 2
$IPT -A POSTROUTING -t mangle -o $DEV -p tcp -m length --length :64 -j MARK --set-mark 10
$IPT -A POSTROUTING -t mangle -o $DEV -p udp --dport 5405 -j MARK --set-mark 10
$IPT -A POSTROUTING -t mangle -o $DEV -p udp --sport 5405 -j MARK --set-mark 10
$IPT -A POSTROUTING -t mangle -o $DEV -p tcp --dport 7788 -j MARK --set-mark 11
$IPT -A POSTROUTING -t mangle -o $DEV -p tcp --sport 7788 -j MARK --set-mark 11
$TC filter add dev $DEV parent 1:0 prio 0 protocol ip handle 10 fw flowid 1:10
$TC filter add dev $DEV parent 1:0 prio 0 protocol ip handle 11 fw flowid 1:11
$TC qdisc add dev $DEV parent 1:10 handle 10: sfq perturb 10
$TC qdisc add dev $DEV parent 1:11 handle 11: sfq perturb 10
$TC qdisc add dev $DEV parent 1:12 handle 12: sfq perturb 10
}}}
Line 1070: Line 1075:
* Dyndns
* Let's encrypt		  * Dyndns
 * Let's encrypt
Line 1078: Line 1083:
{{{
Line 1080: Line 1085:
}}}
Line 1082: Line 1087:
{{{
Line 1084: Line 1089:
}}}
Line 1088: Line 1093:
{{{
Line 1090: Line 1095:
}}}
Line 1092: Line 1097:
{{{
Line 1094: Line 1099:
}}}
Line 1096: Line 1101:
{{{
Line 1098: Line 1103:
}}}

{{{
Line 1103: Line 1110:
}}}
Line 1106: Line 1113:
etc/hosts para todo nodes incluido admin-node master

 10.250.30.202 master.local.site master
 10.250.30.203 node1.local.site node1
 10.250.30.204 node2.local.site node2
 10.250.30.205 node3.local.site node3		 /etc/hosts para todo nodes incluido admin-node master

{{{
10.250.30.202 master.local.site master
10.250.30.203 node1.local.site node1
10.250.30.204 node2.local.site node2
10.250.30.205 node3.local.site node3
}}}
Line 1114: Line 1123:
{{{
Line 1116: Line 1125:
}}}
Line 1118: Line 1127:
{{{
Line 1120: Line 1129:
}}}
Line 1122: Line 1131:
{{{
Line 1124: Line 1133:
}}}
Line 1126: Line 1135:
{{{
Line 1128: Line 1137:
}}}
Line 1133: Line 1142:
== opensuse tumbleweed/leep== == opensuse tumbleweed/leep ==
Line 1136: Line 1145:
  {{{
Line 1138: Line 1147:
}}}

Salida:
{{{
Line 1191: Line 1204:
}}}

Technología de alto disponibilidad y Container LXC y fin de instalación

Como Raspberry Pi está muy popular decidió hacer un proyecto de virtualización con LXC y un NAS con DRBD y iSCSI para el festival de software libre en Ecuador/ Loja en 2018. Un NAS exporta sus discos por red y en LXC aparece los discos si fueran local.

Este instalación se encontra en centros de datos y es interesante para administradores de sistema en Linux.

Este proyecto se puede realizar tambien en KVM, recomiendo KVM nested, en un portátil con un CPU de Intel o AMD, que tienen el feature VMX (Intel) o SMX (AMD).

Como ya hice en arquitectura AMD64 (Intel o AMD) tenía ganas de probar como lo funcione en architectura ARM. Probaré ese instalación para un tiempo de algunas meses. Publicaré un guía sobre el certificado de LPI 304 Virtualización y Alto Disponibilidad ( www.lpi.org ). Lamentablemente esta documentación es en alemán.

Ahora la technología:

NAS DRDB-Cluster:

2 Raspberries con DRBD de modo activo/activo que usa sda2 y sda1 de USB-Stick. DRBD sirve para copiar sus datos al otro Rasperry. Por eso los datos de sda2 son los mismos en dos Raspberries. El sistema es instalado en la tarjeta SD (nombre de SD /dev/ mmcblk...) . DRBD crea un nuevo nombre de disco en ambas Raspberries: drbd0 que es sda2. El modo activo activo de DRBD realicé en la configuración /etc/drbd.conf ( /etc/drbd.d/*.conf es tambien posible) y en pacemaker con la programa crm y corosync. Para crear un iSCSI-Target (=Servidor) usé el software libre istgt cual usa /dev/drbd0 en ambas Rasperries. En LXC-Host los dos iSCSI-Target aparecen como /dev/sda y /dev/sdb.

Container virtualización LXC:

1 Raspberry con programas de LXC para configurar un guest de LXC. El LXC-Host cual usa la tarjeta SD para sus datos (nombre de SD es /dev/ mmcblk...) usa open-iscsi para importar los dos sda2 (= drbd0) del NAS DRBD-Cluster. Los dos sda2 de NAS aparecen en LXC-Host como /dev/sda y /dev/sdb en mi proyecto. Multipathing usa /dev/sda y /dev/sdb en el modo round-robin como /dev/mpatha . Usé /dev/mpatha para LVM-Group cvg1. En consecuencia de Round-Robin y VG cvg1 todos los particiones de LVM están en mpatha, por ejemplo /dev/cvg1/wiki, que significa /etc/cvg1/wiki está en /dev/sda y /dev/sdb, pero /dev/cvg1/wiki está activo solo en una de sda o sdb (Round Robin).

El guest de LXC publicwiki usa /dev/cvg1/wiki para su sistema. Para mi documentación uso mediawiki que es una aplicación en PHP7 que usa apache2 y mariadb. Instalé fio para probar la potencia de su disco, que es en iscsi, drbd y USB-Stick.

Primeramente probé tambien Ceph para un Cluster de NAS, por eso documenté tambien ese proyecto, pero no tenía éxito y tambien ceph necesita un Raspberry más y la carga de Rasperry fuera más larga en ceph que en drbd.

Hardware

  • 3 Raspberry PI 3 B, eth0 @ 100 Mbit/s
  • 2 USB-Sticks 64 GByte (1 USB-Stick de 64 GB para backup)
  • 8-Port Switch 1 GBit/s
  • Fuente de energía: Raspberry original, Samsung, USB-Hub Activo

DRBD

Hardware: 3 Raspberry, ETH0 @100MBit

DRBD, Active/Active, istgt para iscsi en 2 Raspberrys 

 ha-iscsi1(=Hostname), 10.250.30.202
 ha-iscsi2(=Hostname), 10.250.30.203
 Fuente de energía de Raspberry y Samsung
 DRBD: Un USB-Stick de 64 GB el DRBD-Node

LXC Host en un Raspberry 

 lxc1pi (Hostname lxc1), 10.250.30.204
 Fuente de energía USB-Hub activo

LXC Guest 

 wiki

Instalación ha-iscsi

Hosts: ha-iscsi1 y ha-iscsi2, debian stretch (9) 

 apt-get install istgt pacemaker crmsh drbd8-utils vim mtr fio sysbench sysstat tcpdump

Configuración

istgt

/etc/istgt/istgt.conf de ha-iscsi2 

[Global]
 Comment "Global section"
 NodeBase "ha-iscsi2.203.istgt"
 
 PidFile /var/run/istgt.pid
 AuthFile /etc/istgt/auth.conf
 
 MediaDirectory /var/istgt
 
 LogFacility "local7"
 
 Timeout 30
 NopInInterval 20
 
 DiscoveryAuthMethod Auto
 
 MaxSessions 16
 MaxConnections 4
 
 MaxR2T 32
 
 MaxOutstandingR2T 16
 DefaultTime2Wait 2
 DefaultTime2Retain 60
 FirstBurstLength 262144
 MaxBurstLength 1048576
 MaxRecvDataSegmentLength 262144
 
 InitialR2T Yes
 ImmediateData Yes
 DataPDUInOrder Yes
 DataSequenceInOrder Yes
 ErrorRecoveryLevel 0
 
[UnitControl]
  Comment "Internal Logical Unit Controller"
  AuthMethod CHAP Mutual
  AuthGroup AuthGroup10000
  Portal UC1 127.0.0.1:3261
  Netmask 127.0.0.1
 
[PortalGroup1]
  Comment "iscsi portal"
  Portal DA1 10.250.30.203:3260
 
[InitiatorGroup1]
  Comment "Initiator Group1"
  InitiatorName "iqn.xenx.debian:01:4a4e9e22cdb"
  Netmask 10.250.30.0/24
 
[LogicalUnit1]
  Comment "Hard Disk Sample"
  TargetName disk1
  TargetAlias "Data Disk1"
  Mapping PortalGroup1 InitiatorGroup1
  AuthMethod Auto
  AuthGroup AuthGroup1
  UseDigest Auto
  UnitType Disk
 
  LUN0 Storage /dev/drbd0 Auto

Diferencias en ha-iscsi1 istgt.conf: 

 ...
   NodeBase "ha-iscsi1.202.istgt"
 ...
   Portal DA1 10.250.30.202:3260 
 ...

/etc/istgt/auth.conf de ha-iscsi1 y ha-iscsi2 

[AuthGroup1]
  Comment "x.xenx"
  Auth "xenx"  "password" \
 
[AuthGroup9999]
  Auth "xenx"  "password" \
 
[AuthGroup10000]
  Comment "Unit Controller's users"
  Auth "ctluser" "test" "mutualuser" "mutualsecret"
  Auth "onlysingle" "secret"

No systemd controla istgt sino pacemaker. Por lo tanto desactivo istgt para systemd. 

 systemctl disable istgt

DRBD

/etc/drbd.conf de ha-iscsi1 yha-iscsi2 son identicos. 

global {
 dialog-refresh 1;
 minor-count    5;
 usage-count no;
}
common {
}
resource r0 {
 protocol       C;
 handlers {
  split-brain "/usr/lib/drbd/notify-split-brain.sh root";
 }
 disk {
    on-io-error pass_on;
 }
 syncer {
        verify-alg sha1;
        rate 12M;       
 }
 net {
   allow-two-primaries;
   after-sb-0pri discard-younger-primary;
   after-sb-1pri call-pri-lost-after-sb;
   after-sb-2pri call-pri-lost-after-sb;
 }
 startup {
      become-primary-on both;
 }
 on ha-iscsi1 {
    device      /dev/drbd0;
    address     10.250.30.202:7788;
    meta-disk   /dev/sda1[0];
    disk        /dev/sda2;
 }
 on ha-iscsi2 {
    device      /dev/drbd0;
    address     10.250.30.203:7788;
    meta-disk   /dev/sda1[0];
    disk        /dev/sda2;
 }
}

Es una configuración de dos Primaries y pacemaker es para controlar. Uso un propio metadisk con Indexes. La ventaja es que puedo usar el meta disk para otro drbd disk.

En ha-iscsi1 y 2 necesito iniciar drbd ... 

 drbdadm create-md r0

... y activarlas 

 drbdadm up r0

Tengo esta salida de "cat /proc/drbd" 

 ...ds: Inconsistent/inconsistent...

En esta situación no puedo usar drbd. Puedo resolver "inconsistent" con siguiente comando en ha-iscsi1: 

 drbdadm verify r0

Para 20 GByte necesita 20 minutos y después tengo siguiente salida de "cat /proc/drbd" 

 ... ds:UpToDate/UpToDate ...

Corosync y Pacemaker

Primero corosync necesita un llave que produzco con ... 

 corosync-keygen

Después tengo el archivo /etc/corosync/auth.key que copio al otro Raspberry.

/etc/corosync/corosync.conf de ha-iscsi1 y 2 

totem {
       version: 2
       transport: udpu
       cluster_name: ha-iscsi
 
       token: 3000
       token_retransmits_before_loss_const: 10
 
       clear_node_high_bit: yes
 
       crypto_cipher: aes256
       crypto_hash: sha1
 
      interface {
               ringnumber: 0
               bindnetaddr: 10.250.30.0
               ttl: 1
       }
 }
logging {
       fileline: off
 
       to_stderr: no
       to_logfile: no
       to_syslog: yes
 
       syslog_facility: daemon
       debug: off
       
       timestamp: on
       logger_subsys {
               subsys: QUORUM
               debug: off
       }
 }
quorum {
       provider: corosync_votequorum
       two_node: 1
 }
nodelist {
       node {
               ring0_addr: 10.250.30.202
       }
       node {
               ring0_addr: 10.250.30.203
       }
 }

Talvez se debe que reiniciar corosync
{{{
 systemctl stop corosync
 systemctl start corosync

Para Pacemaker primeramente reinicio tambien 

 systemctl stop pacemaker
 systemctl start pacemaker

Ahora puedo configurar Pacemaker con crm

Solo se puede configurar drbd como active/active con Pacemaker. drbd-utils no puede lograrlo.

Desactivar stonith 

 crm configure stonith-enabled=false

Configurar y iniciar DRBD Primary/Primary (significa Active/Active) 

 crm configure primitive drbd_r0 ocf:linbit:drbd params drbd_resource="r0" op monitor interval="20" role="Master" timeout="20" op monitor interval="30" role="Slave" timeout="20"
 crm configure ms ms_drbd_r0 drbd_r0 meta resource-stickiness="100" master-max="2" notify="true" interleave="true"

Salida de "cat /proc/drbd" 

 0: cs:Connected ro:Primary/Primary ds:UpToDate/UpToDate C r-----
    ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:20971392

Salida de "crm status" 

 Stack: corosync
 Current DC: ha-iscsi1 (version 1.1.16-94ff4df) - partition with quorum
 Last updated: Sun Mar 25 06:33:25 2018
 Last change: Sun Mar 25 06:27:48 2018 by root via cibadmin on ha-iscsi2
 
 2 nodes configured
 4 resources configured (2 DISABLED)
 
 Online: [ ha-iscsi1 ha-iscsi2 ]
 
 Full list of resources:
 
 Master/Slave Set: ms_drbd_r0 [drbd_r0]
     Masters: [ ha-iscsi1 ha-iscsi2 ]

Ahora la configuración de istgt 

 crm configure primitive istgt lsb:istgt 
 crm configure clone istgt_clone istgt

Salida de "crm status" 

 ...
  Clone Set: istgt_clone [istgt]
     Started: [ ha-iscsi1 ha-iscsi2 ]
 ...

Instalación lxc1pi

 apt install screen nmon ufw libncurses5-dev git bc multipath-tools lxc open-iscsi

Para multipath se debe instalar nuevo kernel sin ellos no puedo usar multipathing. Estos modules faltan:

  • scsi_dh_alua
  • scsi_dh_emc
  • scsi_dh_rdac
  • dm-multipath 

 cd /usr/src
 git clone --depth=1 https://github.com/raspberrypi/linux
 cd linux

 KERNEL=kernel7
 make bcm2709_defconfig
 make menuconfig

En el menu de "make menuconfig" 

 Device Drivers > SCSI device support > SCSI Device Handlers
   <M>   LSI RDAC Device Handler
   < >   HP/COMPAQ MSA Device Handler
   <M>   EMC CLARiiON Device Handler
   <M>   SPC-3 ALUA Device Handler

 Device Drivers > Multiple devices driver support (RAID and LVM)
   <M>     Multipath target

Compilar el kernel 

 make -j4 zImage modules dtbs

Instalar modules 

 make modules_install

Instalar nuevo kernel 

 cp arch/arm/boot/dts/*.dtb /boot/
 cp arch/arm/boot/dts/overlays/*.dtb* /boot/overlays/
 cp arch/arm/boot/dts/overlays/README /boot/overlays/
 cp arch/arm/boot/zImage /boot/$KERNEL.img

Reiniciar sistema 

 shutdown -r now

Configuración

Firewall

Proteger contra tráfico del router (igual internet) el servicio ssh. 

 ufw default deny incoming
 ufw default allow outgoing

 ufw deny from 10.250.30.250 to any port 22
 ufw allow from 10.250.30.0/24 to any port 22

open-iscsi

Pone nombre de Initiator que se puede encontrar en ha-iscsi1 y2 en el archivo /etc/istgt/istgt.conf 

 # etc/iscsi/initiatorname.iscsi
 InitiatorName=iqn.xenx.debian:01:4a4e9e22cdb

/etc/iscsi/iscsid.conf 

iscsid.startup = /usr/sbin/iscsid
node.startup = automatic
node.startup = manual
node.leading_login = No
node.session.timeo.replacement_timeout = 120
node.conn[0].timeo.login_timeout = 15
node.conn[0].timeo.logout_timeout = 15
node.conn[0].timeo.noop_out_interval = 5
node.conn[0].timeo.noop_out_timeout = 5
node.session.err_timeo.abort_timeout = 15
node.session.err_timeo.lu_reset_timeout = 30
node.session.err_timeo.tgt_reset_timeout = 30
node.session.initial_login_retry_max = 8
node.session.cmds_max = 128
node.session.queue_depth = 32
node.session.xmit_thread_priority = -20
node.session.iscsi.InitialR2T = No
node.session.iscsi.ImmediateData = Yes
node.session.iscsi.FirstBurstLength = 262144
node.session.iscsi.MaxBurstLength = 16776192

Busqando de ha-iscsi1 y 2, login y inicio automático

ha-iscsi1 

 iscsiadm -m discovery -t st -p 10.250.30.202
 iscsiadm --mode node --targetname ha-iscsi1.202.istgt:disk1 --portal 10.250.30.202:3260 --login
 iscsiadm -m node -T ha-iscsi1.202.istgt:disk1 --portal 10.250.30.202:3260 --op update -n node.startup -v automatic

ha-iscsi2 

 iscsiadm -m discovery -t st -p 10.250.30.203
 iscsiadm --mode node --targetname ha-iscsi2.203.istgt:disk1 --portal 10.250.30.203:3260 --login
 iscsiadm -m node -T ha-iscsi2.203.istgt:disk1 --portal 10.250.30.203:3260 --op update -n node.startup -v automatic

Probar con "cat /proc/partitions" 

    ...
    8        0   20971520 sda
    8       16   20971520 sdb
    ...

multipath-tools

/etc/multipath.conf 

blacklist {
devnode "ram"
devnode "mmcblk0"
devnode "sdc"
}
 
defaults {
user_friendly_names yes
path_selector "round-robin 0"
}

Reiniciar mutlipath-tools 

 systemctl stop multipath-tools
 systemctl start multipath-tools

Probar con "lsblk" 

 NAME                  MAJ:MIN RM  SIZE RO TYPE  MOUNTPOINT
 sda                     8:0    0   20G  0 disk  
 └─mpatha              254:0    0   20G  0 mpath 
 sdb                     8:16   0   20G  0 disk  
 └─mpatha              254:0    0   20G  0 mpath

Instalación de primer container publicwiki

 pvcreate /dev/mapper/mpatha
 vgcreate cvg1 /dev/mapper/mpatha

Crear un LXC-Container 

 lxc-create -t debian -n wiki -B lvm --vgname cvg1 --fssize 8G --fstype ext4

Ahora lxc1 voy a descargar un sistema completo de 211 MByte. Se puede encontrar en: 

 /var/cache/lxc/debian/rootfs-stable-armhf

Se crea un nuevo container con ese commando ... 

 lxc-create -t debian -n proxy -B lvm --vgname cvg1 --fssize 8G --fstype ext4

... lxc usa el sistema de /var/cache/lxc/debian/rootfs-stable-armhf

La Configuración de wiki container se encuentra aqui: 

 /var/lib/lxc/wiki/config

Contenido 

#lxc.include = /usr/share/lxc/config/nesting.conf
# (Be aware this has security implications)
 
lxc.network.type = veth
lxc.network.flags = up
lxc.network.link = br0 
lxc.network.ipv4 = 10.250.30.206/24
lxc.network.ipv4.gateway = 10.250.30.4
lxc.rootfs = /dev/cvg1/wiki
lxc.rootfs.backend = lvm
 
# Common configuration
lxc.include = /usr/share/lxc/config/debian.common.conf
 
# Container specific configuration
lxc.tty = 4
lxc.utsname = wiki
lxc.arch = armhf

Instalaciones en wiki 

 apt vim fio apache2 mediawiki php mariadb-server gnutls-bin ufw

Configuración mediawiki en Container wiki

/etc/mediawiki/LocalSettings.php

Restrictar derechos 

 ...
# prevent editing and reading by anons (except for exception listed below):
$wgGroupPermissions['*']['edit'] = false;
$wgGroupPermissions['*']['createaccount'] = false;
$wgGroupPermissions['*']['read'] = true;
 
# same for normal registered users:
$wgGroupPermissions['user']['edit'] = false;
$wgGroupPermissions['user']['read'] = true;
 
# allow everyone read access to these pages:
#$wgWhitelistRead = array( "Main Page", "Public stuff" );
 
# allow sysops to read and edit normally: 
$wgGroupPermissions['sysop']['edit'] = true;
$wgGroupPermissions['sysop']['read'] = true;

Visual Editor 

...
wfLoadExtension( 'VisualEditor' );
 
// Enable by default for everybody
$wgDefaultUserOptions['visualeditor-enable'] = 1;
// Optional: Set VisualEditor as the default for anonymous users
// otherwise they will have to switch to VE
$wgDefaultUserOptions['visualeditor-editor'] = "visualeditor";
 
// Don't allow users to disable it
$wgHiddenPrefs[] = 'visualeditor-enable';
....

Activar cargar 

...
$wgEnableUploads = true;
...

Configuración Apache2 en Container wiki

Cambio este archivo: 

 /etc/apache2/sites-enabled/000-default.conf

Explicaciónes están en los commentarios 

 ...
<Virtualhost ...
        ...
        # Custom Configs, Redirect to mediawiki-app
        RewriteEngine   on
        RewriteRule     ^/$ "/mediawiki/" [R]
        ## Against DDOS-Attacks ##
        # https://httpd.apache.org/docs/2.4/mod/mod_reqtimeout.html#requestreadtimeout
        # 5/10 seconds (Header/Body)  and every 1000 Bytes one second more
        RequestReadTimeout header=5,MinRate=1000 body=10,MinRate=1000
        # Timeout für Get/Post/Put-Requests, ACKs bei TCP-Paketen
        TimeOut 10
        # Einige Limits für Body und Header #
        # Upload bis 10 MByte
        LimitRequestBody 10485760
        # Maximale Anzahl Request-Header je Client, default 100
        LimitRequestFields 20
        # Grösse des Reqest-Headers in Bytes, default 8190
        LimitRequestFieldSize 2048 
        # Grösse der Anfragezeile des Clients in Bytes, default 8190
        LimitRequestLine 512
        # XML- Request Body in Bytes; default 1000000
        LimitXMLRequestBody 100000
</Virtualhost>

Configuración Firewall en Container wiki

En este sistema el firewall está muy restrictivia. Por eso niego el tráfico saliendo.

Tambien di cuenta que tráfico del router 10.250.30.250 no puede atacar por ssh. 

 ufw default deny incoming
 ufw default deny outgoing
 ufw deny from 10.250.30.250 to any port 22
 ufw allow from 10.250.30.0/24 to any port 22
 ufw allow http
 ufw allow https
 ufw allow out to any port 53
 ufw allow out to any port 80
 ufw allow out to any port 443

Para probar el firewall por 30 secundos: 

 ufw enable; sleep 30; ufw disable

Configuración SSL para Apache2 en Container wiki

No está. ¿Let's encrypt?

Operaciónes

Apagar Cluster

En este orden

  • lxc1pi
  • ha-iscsi1
  • ha-iscsi2

lxc1pi

* Apager Containers 

 lxc-stop -n wiki

Se prueba con ... 

 lxc-ls --fancy

* Apagar sistema 

 shutdown -h now

ha-iscsi1 o ha-iscsi2

  • Apagar istgt 

 crm resource stop istgt
  • Apagar drbd 

 crm resource stop drbd_r0

* Apagar sistema 

 shutdown -h now

Encender Cluster

En este orden

  • ha-iscsi1 y ha-iscsi2
  • lxc1pi

=== ha-iscsi1 o 2===

  • Iniciar drbd 

 crm resource start drbd_r0
  • iniciar istgt 

 crm resource start istgt

Probar con ... 

 crm status

y ... 

 cat /proc/drbd

y ... 

 netstat -ntul | grep 3260

lxc1pi

¡Encienda lxc1pi sin USB-stick!

Se prueba que iscsi sda y sdb existen 

 cat /proc/partitions

y ... 

 lsblk

y 

 lvs

Iniciar Container wiki 

 lxc-start -n wiki

Recuperar cluster fallado (Workaround)

Problema: Si uno apaga uno de los drbd-nodes y encende este drbd-node, los drbd-nodes iniciar a sync sus datos. Pero el sync no completa ( cat /prod/drbd -> stalled).

Node lxc1pi está apagado

Desactiva istgt y drbd_r0 

 crm resource stop istgt
 crm resource stop drbd_r0

Pare pacemaker en ambas nodes 

 systemctl stop pacemaker

Borra meta-disk en ambas nodes 

 drbdadm down r0
 drbdadm wipe-md r0

Iniciar drbd de nuevo

En ambas nodes 

 drbdadm create-md r0
 drbdadm up r0

En unos de nodes 

 drbdadm verify r0

Puede ser que uno o dos Raspberrys pierden sus USB-Sticks. Por eso de nuevo meta los USB-Sticks.

Iniciar pacemaker 

 systemctl start pacemaker

Espera algunos minutos 

 crm resource start drbd_r0

Espera algunos minutos 

 crm resource start istgt

Después encienda lxc1pi

Prueba si lvm wiki está existiendo en lxc1pi 

 lvs

En mi caso asi es 

 LV   VG   Attr       LSize Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
  wiki cvg1 -wi-a----- 8.00g

Fallado cluster con drbd standalone

Si quieres hacer la recuperación otra vez puede usar drbd standalone en el node sano, "UpToDate".

Primeramente apaga los containers en lxc1pi y mismo.

Solo "shutdown -h now" en el node "inconsistente" o desconectar el corriente de node malo.

Si el node sano ya perdió el disco fisico, apagar el node malo. Ademas, en el node sano, apagar los servicios con crm, luego apager pacemaker con systemctl y iniciar drbd manualmente: 

 drbdadm up r0
 drbdadm primary r0
 systemctl start istgt

Después encienda lxc1pi y sus containers.

crm: Borrar mensajes de error

ha-iscsi1 y ha-iscsi2 

 crm_resource --cleanup --node ha-iscsi1 --resource drbd_r0
 crm_resource --cleanup --node ha-iscsi1 --resource istgt
 crm_resource --cleanup --node ha-iscsi2 --resource drbd_r0
 crm_resource --cleanup --node ha-iscsi2 --resource istgt

Se puede usar estes comandos aunque no haya errores.

Causas pueden ser que el usb-stick desaparecer en Linux

Backup de LVM de lxc-guest publicwiki con lxc-Host lxc1pi

Para backups tengo otro USB-Stick.

El LXC-Host tiene por iscsi el LVM /dev/cvg1/wiki y tiene 8 GB. Por eso conecto el USB-Stick en Raspberry de LXC y crear otro vg 

 pvcreate /dev/sdc
 vgcreate drbdbackup /dev/sdc
 lvcreate -L +8G -n wiki drbdbackup

lvscan 

  ACTIVE            '/dev/drbdbackup/wiki' [8.00 GiB] inherit
  ACTIVE            '/dev/cvg1/wiki' [8.00 GiB] inherit

Con el siguente comando se escribe datos de /dev/cvg1/wiki en /dev/drbdbackup/wiki 

 dd if=/dev/cvg1/wiki of=/dev/drbdbackup/wiki bs=4096 status=progress

Para recuperar: 

 dd if=/dev/drbdbackup/wiki of=/dev/cvg1/wiki bs=4096 status=progress

DRBD Probar potencia de disco duro en LXC guest wiki: Datos random, escribiendo

 fio --name fio_test_file --direct=1 --rw=randwrite --bs=4k --size=1G --numjobs=4 --time_based --runtime=600 --group_reporting

Resultado IOPS=90 

 fio-2.16
 Starting 4 processes
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 Jobs: 4 (f=4): [w(4)] [5.2% done] [0KB/0KB/0KB /s] [0/0/0 iops] [eta 03h:03m:59s]   
 fio_test_file: (groupid=0, jobs=4): err= 0: pid=744: Mon Apr  9 02:09:20 2018
  write: io=218920KB, bw=368930B/s, iops=90, runt=607633msec
    clat (msec): min=1, max=85404, avg=44.37, stdev=1583.74
     lat (msec): min=1, max=85404, avg=44.38, stdev=1583.74
    clat percentiles (usec):
     |  1.00th=[ 1656],  5.00th=[ 1720], 10.00th=[ 1752], 20.00th=[ 1800],
     | 30.00th=[ 1848], 40.00th=[ 1912], 50.00th=[ 1960], 60.00th=[ 2040],
     | 70.00th=[ 2096], 80.00th=[ 2224], 90.00th=[ 2448], 95.00th=[ 2704],
     | 99.00th=[ 3440], 99.50th=[ 4640], 99.90th=[7831552], 99.95th=[16711680],
     | 99.99th=[16711680]
    lat (msec) : 2=55.49%, 4=43.87%, 10=0.36%, 20=0.04%, 50=0.02%
    lat (msec) : 100=0.07%, 250=0.03%, >=2000=0.13%
  cpu          : usr=0.07%, sys=0.69%, ctx=65968, majf=0, minf=81
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=54730/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
 
 Run status group 0 (all jobs):
  WRITE: io=218920KB, aggrb=360KB/s, minb=360KB/s, maxb=360KB/s, mint=607633msec, maxt=607633msec
 
 Disk stats (read/write):
    dm-1: ios=2/57656, merge=0/0, ticks=10/8063580, in_queue=8794040, util=100.00%, aggrios=2/54916, aggrmerge=0/2752, aggrticks=10/4883320, aggrin_queue=5059740,  aggrutil=100.00%
    dm-0: ios=2/54916, merge=0/2752, ticks=10/4883320, in_queue=5059740, util=100.00%, aggrios=16/27453, aggrmerge=0/0, aggrticks=249775/2020915, aggrin_queue=2358900, aggrutil=60.84%
  sdb: ios=20/29301, merge=0/0, ticks=278230/2786920, in_queue=3241630, util=60.84%
  sda: ios=13/25606, merge=0/0, ticks=221320/1254910, in_queue=1476170, util=39.36%

DRBD: Probar potencia de disco duro en LXC guest wiki: Datos random, leyendo

 fio --name fio_test_file --direct=1 --rw=randread --bs=4k --size=1G --numjobs=4 --time_based --runtime=600 --group_reporting

Resultado: IOPS=487 

 fio-2.16
 Starting 4 processes
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 Jobs: 4 (f=4): [r(4)] [100.0% done] [1704KB/0KB/0KB /s] [426/0/0 iops] [eta 00m:00s]
 fio_test_file: (groupid=0, jobs=4): err= 0: pid=753: Mon Apr  9 02:49:54 2018
  read : io=1143.1MB, bw=1950.9KB/s, iops=487, runt=600009msec
    clat (msec): min=1, max=415, avg= 8.17, stdev= 2.92
     lat (msec): min=1, max=415, avg= 8.18, stdev= 2.92
    clat percentiles (usec):
     |  1.00th=[ 5088],  5.00th=[ 5920], 10.00th=[ 6368], 20.00th=[ 6944],
     | 30.00th=[ 7392], 40.00th=[ 7776], 50.00th=[ 8160], 60.00th=[ 8512],
     | 70.00th=[ 8896], 80.00th=[ 9280], 90.00th=[ 9792], 95.00th=[10432],
     | 99.00th=[11840], 99.50th=[12864], 99.90th=[16320], 99.95th=[17536],
     | 99.99th=[22400]
    lat (msec) : 2=0.01%, 4=0.11%, 10=91.65%, 20=8.23%, 50=0.01%
    lat (msec) : 500=0.01%
  cpu          : usr=0.30%, sys=0.84%, ctx=297716, majf=0, minf=74
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=292631/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
 
 Run status group 0 (all jobs):
   READ: io=1143.1MB, aggrb=1950KB/s, minb=1950KB/s, maxb=1950KB/s, mint=600009msec, maxt=600009msec
 
 Disk stats (read/write):
    dm-1: ios=292556/5, merge=0/0, ticks=2353450/40, in_queue=2353820, util=100.00%, aggrios=292631/5, aggrmerge=0/1, aggrticks=2368930/40, aggrin_queue=2368520, aggrutil=100.00%
    dm-0: ios=292631/5, merge=0/1, ticks=2368930/40, in_queue=2368520, util=100.00%, aggrios=146316/2, aggrmerge=0/0, aggrticks=1172205/5, aggrin_queue=1172085, aggrutil=100.00%
  sdb: ios=292633/4, merge=0/0, ticks=2344410/10, in_queue=2344170, util=100.00%
  sda: ios=0/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%

USB-Stick solo: Probar potencia de disco duro: Datos sequencial, escribiendo

Con Cache 

 dd if=/dev/zero of=zerofile bs=1M count=2048

Resultado 

 2147483648 bytes (2.1 GB, 2.0 GiB) copied, 126.028 s, 17.0 MB/s

Sin Cache 

  dd if=/dev/zero of=zerofile bs=1M count=2048 oflag=direct

Resultado 

 2147483648 bytes (2.1 GB, 2.0 GiB) copied, 107.316 s, 20.0 MB/s

USB-Stick solo: Probar potencia de disco duro: Datos random, escribiendo

 fio --name fio_test_file --direct=1 --rw=randwrite --bs=4k --size=1G --numjobs=4 --time_based --runtime=600 --group_reporting

Resultado: IOPS=92 

 fio_test_file: (g=0): rw=randwrite, bs=4K-4K/4K-4K/4K-4K, ioengine=sync, iodepth=1
 ...
 fio-2.2.10
 Starting 4 processes
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 Jobs: 4 (f=4): [w(4)] [5.4% done] [0KB/0KB/0KB /s] [0/0/0 iops] [eta 02h:58m:59s] 
 fio_test_file: (groupid=0, jobs=4): err= 0: pid=2776: Wed Mar 14 18:02:10 2018
  write: io=224596KB, bw=378306B/s, iops=92, runt=607937msec
    clat (msec): min=2, max=29861, avg=43.28, stdev=471.46
     lat (msec): min=2, max=29861, avg=43.28, stdev=471.46
    clat percentiles (msec):
     |  1.00th=[   25],  5.00th=[   26], 10.00th=[   26], 20.00th=[   27],
     | 30.00th=[   28], 40.00th=[   28], 50.00th=[   28], 60.00th=[   29],
     | 70.00th=[   29], 80.00th=[   30], 90.00th=[   30], 95.00th=[   31],
     | 99.00th=[   44], 99.50th=[   73], 99.90th=[ 8225], 99.95th=[ 9503],
     | 99.99th=[16712]
    bw (KB  /s): min=    0, max=  164, per=37.09%, avg=136.85, stdev=24.83
    lat (msec) : 4=0.01%, 10=0.01%, 20=0.07%, 50=99.05%, 100=0.58%
    lat (msec) : 250=0.09%, 500=0.02%, 750=0.02%, >=2000=0.15%
  cpu          : usr=0.07%, sys=0.32%, ctx=56830, majf=0, minf=79
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=56149/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
 
 Run status group 0 (all jobs):
  WRITE: io=224596KB, aggrb=369KB/s, minb=369KB/s, maxb=369KB/s, mint=607937msec, maxt=607937msec
 
 Disk stats (read/write):
  sda: ios=0/58010, merge=0/21112, ticks=0/6911164, in_queue=7051628, util=100.00%

USB-Stick solo: Probar potencia de disco duro: Datos random, leyendo

 fio --name fio_test_file --direct=1 --rw=randread --bs=4k --size=1G --numjobs=4 --time_based --runtime=600 --group_reporting

Resultado IOPS=677 

 fio_test_file: (g=0): rw=randread, bs=4K-4K/4K-4K/4K-4K, ioengine=sync, iodepth=1
 ...
 fio-2.2.10
 Starting 4 processes
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 fio_test_file: Laying out IO file(s) (1 file(s) / 1024MB)
 Jobs: 4 (f=4): [r(4)] [100.0% done] [2718KB/0KB/0KB /s] [679/0/0 iops] [eta 00m:00s]
 fio_test_file: (groupid=0, jobs=4): err= 0: pid=3482: Wed Mar 14 18:33:42 2018
  read : io=1588.3MB, bw=2710.6KB/s, iops=677, runt=600006msec
    clat (msec): min=2, max=1794, avg= 5.88, stdev=13.72
     lat (msec): min=2, max=1794, avg= 5.88, stdev=13.72
    clat percentiles (usec):
     |  1.00th=[ 4768],  5.00th=[ 4960], 10.00th=[ 5088], 20.00th=[ 5216],
     | 30.00th=[ 5280], 40.00th=[ 5408], 50.00th=[ 5600], 60.00th=[ 5792],
     | 70.00th=[ 6048], 80.00th=[ 6304], 90.00th=[ 6624], 95.00th=[ 7008],
     | 99.00th=[ 7648], 99.50th=[ 7904], 99.90th=[ 8512], 99.95th=[ 8640],
     | 99.99th=[13888]
    bw (KB  /s): min=   42, max=  731, per=25.39%, avg=688.09, stdev=52.28
    lat (msec) : 4=0.01%, 10=99.99%, 20=0.01%, 50=0.01%, 2000=0.01%
  cpu          : usr=0.33%, sys=1.04%, ctx=407412, majf=0, minf=73
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=406591/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1
 
 Run status group 0 (all jobs):
   READ: io=1588.3MB, aggrb=2710KB/s, minb=2710KB/s, maxb=2710KB/s, mint=600006msec, maxt=600006msec
 
 Disk stats (read/write):
  sda: ios=406550/4, merge=0/1, ticks=2372436/84, in_queue=2371860, util=100.00%

Troubleshooting

No sufficiente Volt significa muchas simtomas

Solución: Fuente de corriente de Raspberry Foundation o de Samsung

Simtomas:

  • DRBD-Nodes: Muchos IO-Errors en sda. Los Observé con "dmesg"
  • LXC-guest: Rootfs en Ext4 está read only.

  • DRBD-Node: uno es "inconsistent". Lo observé con "cat /proc/drbd". "UpToDate" es normal.

Como realizé lo que no hay suficiente Volt: 

 /var/log/kern.log
 ...
 ... Voltage normalised ...
 ... Under-voltage detected ...

Split-Brain en DRBD

Causa: Ambas DRBD-Nodes no puede communicar en Port 7788. Mucho Traffico en iscsi, port 3260, y drbd, port 7788, porque probé la potencia de discos con fio y dd. El tráfico de iscsi pusiera ser tan alto. Supongo que en consecuencia los dos drbd nodes no pusieran contactarse en port 7788.

Otra causa posible: No hay suficiente Volt.

SImtoma: Casi siempre drbd no funcionó cuando probaba la potencia de discos con fio o dd.

Solución: Traffic shaping con este Skript en /root/tc.sh en ha-iscsi1 y ha-iscsi2 

DEV=eth0
IPT=/sbin/iptables
TC=/sbin/tc
$IPT -t mangle -F
$TC qdisc del dev $DEV ingress > /dev/null 2>&1
$TC qdisc del dev $DEV root > /dev/null 2>&1
$TC qdisc del dev lo root > /dev/null 2>&1
$TC qdisc add dev $DEV root handle 1:0 htb default 12 r2q 6
$TC class add dev $DEV parent 1:0 classid 1:1 htb rate 100mbit ceil 100mbit
$TC class add dev $DEV parent 1:1 classid 1:10 htb rate  10mbit ceil 100mbit prio 0
$TC class add dev $DEV parent 1:1 classid 1:11 htb rate  50mbit ceil 100mbit prio 1
$TC class add dev $DEV parent 1:1 classid 1:12 htb rate  40mbit ceil 100mbit prio 2
$IPT -A POSTROUTING -t mangle -o $DEV -p tcp -m length --length :64 -j MARK --set-mark 10
$IPT -A POSTROUTING -t mangle -o $DEV -p udp --dport 5405 -j MARK --set-mark 10
$IPT -A POSTROUTING -t mangle -o $DEV -p udp --sport 5405 -j MARK --set-mark 10
$IPT -A POSTROUTING -t mangle -o $DEV -p tcp --dport 7788 -j MARK --set-mark 11
$IPT -A POSTROUTING -t mangle -o $DEV -p tcp --sport 7788 -j MARK --set-mark 11
$TC filter add dev $DEV parent 1:0 prio 0 protocol ip handle 10 fw flowid 1:10
$TC filter add dev $DEV parent 1:0 prio 0 protocol ip handle 11 fw flowid 1:11
$TC qdisc add dev $DEV parent 1:10 handle 10: sfq perturb 10
$TC qdisc add dev $DEV parent 1:11 handle 11: sfq perturb 10 
$TC qdisc add dev $DEV parent 1:12 handle 12: sfq perturb 10

Que significa eso:

* Respuesta de TCP tiene la más alta prioridad y corosync (prio 0 y marca de mangletable 10). 10 MBit velocidad. * Port de DRBD tiene la segunda más alta prioridad y corosync (prio 1 y marca de mangletable 11). 50 MBit velocidad. * Otro Ports y tambien iscsi, 3260, tienen la tercera más prioridad (prio 2 y marca de tc "default 12"). 40 MBit velocidad.

Resync de DRBD está "stalled"

Causa: Apago un drbd-node para mantener. El sync funcionó para aproxidamente 8 GByte (a veces más, a veces menos)

Workaround: en este documento: "Recuperar cluster fallado (Workaround)"

Solución posible: Disco de DRBD con propia fuente de corriente y USB. Carcasa de disco 3,5 " . ( Carcasa de disco 2,5 " normalmente no tiene propia fuente de corriente)

To_Do

  • Dyndns
  • Let's encrypt

Ceph

Ubuntu

Desactivar servicios 

 systemctl disable cloud-config.service cloud-final.service cloud-init-local.service cloud-init.service iscsi.service iscsid.service lvm2-monitor.service lxcfs.service lxd-containers.service open-iscsi.service  unattended-upgrades.service

Instalar sysstat para monitor sistema y fio para probar velocidad de disco duro 

 apt install sysstat fio sysbench

Para systat se cambia /etc/default/sysstat de "false" a "true"

Probar con ... 

 sar -A

Nuevo usario para Ceph 

 useradd -m master -S /bin/bash -G sudo

Cambiar /etc/sudo con visudo 

 visudo

 ...
 #%sudo   ALL=(ALL:ALL) ALL
 %sudo   ALL=NOPASSWD: ALL
 ...

Cambiar /etc/hostname a master (node1, node2, node3)

/etc/hosts para todo nodes incluido admin-node master 

10.250.30.202   master.local.site master
10.250.30.203   node1.local.site node1
10.250.30.204   node2.local.site node2
10.250.30.205   node3.local.site node3

En mi portátil disponer accesso al admin-node master 

 ssh-copy-id master@10.250.30.202

con2master es un alias en ~/.bashrc "alias con2master='ssh master@10.250.30.202'". Con ese commando pruebo el accesso 

 con2master

En Raspberry crear un clave para ssh con usario master 

 ssh-keygen

Copiar clave público a /home/master/.ssh/authorized_keys 

 ssh-copy-id master@master

Cancelar el proyecto con Ubuntu

Causo: Ceph.com no dispone paquetes para armhf, sino para aarm64. Ubuntu no dispone una distribución para Raspberry en arquitectura aarm64, sino solo armhf. Por eso no tenía un paquete "ceph-mon" o "ceph-osd".

opensuse tumbleweed/leep

ceph-mon cancelar con mensaje "illegal instruction" 

 node1:/etc/ceph # /usr/bin/ceph-mon -f --cluster ceph --id node1 --setuser ceph --setgroup ceph

Salida: 

 *** Caught signal (Illegal instruction) **
 in thread ffffac027ee0 thread_name:msgr-worker-2
 ceph version 13.0.1-2078-g85df538001 (85df5380012dc6b0c18645c783d428d6e8a9fc06) mimic (dev)
 1: (()+0x41eba0) [0xaaaaea34fba0]
 2: (__kernel_rt_sigreturn()+0) [0xffffbea0766c]
 3: (ceph_crc32c_aarch64()+0x4a0) [0xffffb662c5d8]
 4: (ceph::buffer::list::crc32c(unsigned int) const+0xdc) [0xffffb61e3184]
 5: (Message::encode(unsigned long, int)+0x108) [0xffffb62a4200]
 6: (AsyncConnection::prepare_send_message(unsigned long, Message*, ceph::buffer::list&)+0x1b0) [0xffffb6336ee8]
 7: (AsyncConnection::handle_write()+0x26c) [0xffffb633cc4c]
 8: (EventCenter::process_events(int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)+0xacc) [0xffffb63572e4]
 9: (()+0x3ad35c) [0xffffb635a35c]
 10: (()+0xc2d4c) [0xffffb5d69d4c]
 11: (()+0x705c) [0xffffb5f7f05c]
 12: (()+0xd3e3c) [0xffffb5b02e3c]
 2018-03-18 03:23:31.062 ffffac027ee0 -1 *** Caught signal (Illegal instruction) **
 in thread ffffac027ee0 thread_name:msgr-worker-2
 
 ceph version 13.0.1-2078-g85df538001 (85df5380012dc6b0c18645c783d428d6e8a9fc06) mimic (dev)
 1: (()+0x41eba0) [0xaaaaea34fba0]
 2: (__kernel_rt_sigreturn()+0) [0xffffbea0766c]
 3: (ceph_crc32c_aarch64()+0x4a0) [0xffffb662c5d8]
 4: (ceph::buffer::list::crc32c(unsigned int) const+0xdc) [0xffffb61e3184]
 5: (Message::encode(unsigned long, int)+0x108) [0xffffb62a4200]
 6: (AsyncConnection::prepare_send_message(unsigned long, Message*, ceph::buffer::list&)+0x1b0) [0xffffb6336ee8]
 7: (AsyncConnection::handle_write()+0x26c) [0xffffb633cc4c]
 8: (EventCenter::process_events(int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)+0xacc) [0xffffb63572e4]
 9: (()+0x3ad35c) [0xffffb635a35c]
 10: (()+0xc2d4c) [0xffffb5d69d4c]
 11: (()+0x705c) [0xffffb5f7f05c]
 12: (()+0xd3e3c) [0xffffb5b02e3c]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
 
     0> 2018-03-18 03:23:31.062 ffffac027ee0 -1 *** Caught signal (Illegal instruction) **
 in thread ffffac027ee0 thread_name:msgr-worker-2
 
 ceph version 13.0.1-2078-g85df538001 (85df5380012dc6b0c18645c783d428d6e8a9fc06) mimic (dev)
 1: (()+0x41eba0) [0xaaaaea34fba0]
 2: (__kernel_rt_sigreturn()+0) [0xffffbea0766c]
 3: (ceph_crc32c_aarch64()+0x4a0) [0xffffb662c5d8]
 4: (ceph::buffer::list::crc32c(unsigned int) const+0xdc) [0xffffb61e3184]
 5: (Message::encode(unsigned long, int)+0x108) [0xffffb62a4200]
 6: (AsyncConnection::prepare_send_message(unsigned long, Message*, ceph::buffer::list&)+0x1b0) [0xffffb6336ee8]
 7: (AsyncConnection::handle_write()+0x26c) [0xffffb633cc4c]
 8: (EventCenter::process_events(int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)+0xacc) [0xffffb63572e4]
 9: (()+0x3ad35c) [0xffffb635a35c]
 10: (()+0xc2d4c) [0xffffb5d69d4c]
 11: (()+0x705c) [0xffffb5f7f05c]
 12: (()+0xd3e3c) [0xffffb5b02e3c]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
 
 
 Ungültiger Maschinenbefehl (core dumped)

Según investigación probablemente será que ceph no es compatible con raspberry pi

last edited 2018-05-08 23:04:17 by Michael Roessler