From 346b61597b3fb2da4f6082b120856f5999d895f2 Mon Sep 17 00:00:00 2001 From: Adam Parler Date: Mon, 22 Apr 2024 00:46:15 -0700 Subject: [PATCH] Added BeeGFS config. Moved env-modules to modules --- scripts/stage0/install_beegfs.sh | 101 ++++++++++++++++++ ...tall_env-modules.sh => install_modules.sh} | 5 + scripts/stage0/install_stage0.sh | 3 +- 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100755 scripts/stage0/install_beegfs.sh rename scripts/stage0/{install_env-modules.sh => install_modules.sh} (89%) diff --git a/scripts/stage0/install_beegfs.sh b/scripts/stage0/install_beegfs.sh new file mode 100755 index 0000000..348e248 --- /dev/null +++ b/scripts/stage0/install_beegfs.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# node01: Management Server +# node02: Metadata Server +# node03: Storage Server +# node04: Client + +alias node01='172.17.2.101' +alias node02='172.17.2.102' +alias node03='172.17.2.103' +alias node04='172.17.2.104' + +for ((i = 1 ; i < 5 ; i++)); do + ssh-copy-id -i ~/.ssh/id_rsa root@node0$i + ssh root@node0$i + yum install -y wget fio + hostnamectl set-hostname node0$i + wget -O /etc/yum.repos.d/beegfs_rhel8.repo https://www.beegfs.io/release/beegfs_7.4.3/dists/beegfs-rhel8.repo + + systemctl disable firewalld + systemctl stop firewalld + sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux + sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config + + cat <> /etc/hosts + +10.0.1.2 node01 +10.0.1.3 node02 +10.0.1.4 node03 +10.0.1.5 node04 +EOF + reboot now +done + + +ssh root@node01 yum install -y beegfs-mgmtd # management service +ssh root@node02 yum install -y beegfs-meta #libbeegfs-ib # metadata service; libbeegfs-ib is only required for RDMA +ssh root@node03 yum install -y beegfs-storage #libbeegfs-ib # storage service; libbeegfs-ib is only required for RDMA +ssh root@node04 yum install -y kernel-devel beegfs-client beegfs-helperd beegfs-utils # client and command-line utils + +ssh root@node01 +/opt/beegfs/sbin/beegfs-setup-mgmtd -p /data/beegfs/beegfs_mgmtd +systemctl start beegfs-mgmtd +systemctl status beegfs-mgmtd + +ssh root@node02 +/opt/beegfs/sbin/beegfs-setup-meta -p /data/beegfs/beegfs_meta -s 2 -m node01 +systemctl start beegfs-meta +systemctl status beegfs-meta + +ssh root@node03 +mkdir -p /data/beegfs_storage{1..3} +mkfs.ext4 /dev/disk/by-id/scsi-0Linode_Volume_node01 +mkfs.ext4 /dev/disk/by-id/scsi-0Linode_Volume_node02 +mkfs.ext4 /dev/disk/by-id/scsi-0Linode_Volume_node03 + +mount /dev/disk/by-id/scsi-0Linode_Volume_node01 /data/beegfs_storage1 +mount /dev/disk/by-id/scsi-0Linode_Volume_node02 /data/beegfs_storage2 +mount /dev/disk/by-id/scsi-0Linode_Volume_node03 /data/beegfs_storage3 + +cat "/dev/disk/by-id/scsi-0Linode_Volume_node01 /data/beegfs_storage1" >> /etc/fstab +cat "/dev/disk/by-id/scsi-0Linode_Volume_node02 /data/beegfs_storage2" >> /etc/fstab +cat "/dev/disk/by-id/scsi-0Linode_Volume_node03 /data/beegfs_storage3" >> /etc/fstab + +/opt/beegfs/sbin/beegfs-setup-storage -p /data/beegfs_storage1 -s 3 -i 301 -m node01 +/opt/beegfs/sbin/beegfs-setup-storage -p /data/beegfs_storage2 -s 3 -i 302 +/opt/beegfs/sbin/beegfs-setup-storage -p /data/beegfs_storage3 -s 3 -i 303 + +systemctl start beegfs-storage +systemctl status beegfs-storage + +ssh root@node04 +/opt/beegfs/sbin/beegfs-setup-client -m node01 +systemctl start beegfs-helperd +systemctl status beegfs-helperd +systemctl start beegfs-client +systemctl status beegfs-client +vim /etc/beegfs/beegfs-mounts.conf + +# set connDisableAuthentication to true on all nodes. + +ssh node04 +beegfs-ctl --listnodes --nodetype=meta --nicdetails +beegfs-ctl --listnodes --nodetype=storage --nicdetails +beegfs-ctl --listnodes --nodetype=client --nicdetails +beegfs-net # Displays connections the client is actually using +beegfs-check-servers # Displays possible connectivity of the services +beegfs-df # Displays free space and inodes of storage and metadata targets + + +ssh root@node01 +fio --name=test --group_report --filename=/mnt/beegfs/testfile1 --ioengine=posixaio --time_based --runtime=180 --startdelay=10 --nrfiles=1 --size=1G --numjobs=1 --bs=128K --rw=read + +ssh root@node04 +fio --name=test --group_report --filename=/mnt/beegfs/testfile2 --ioengine=posixaio --time_based --runtime=180 --startdelay=10 --nrfiles=1 --size=1G --numjobs=1 --bs=128K --rw=read + + + + + + diff --git a/scripts/stage0/install_env-modules.sh b/scripts/stage0/install_modules.sh similarity index 89% rename from scripts/stage0/install_env-modules.sh rename to scripts/stage0/install_modules.sh index 8a64c41..ee08b75 100644 --- a/scripts/stage0/install_env-modules.sh +++ b/scripts/stage0/install_modules.sh @@ -1,3 +1,8 @@ +#!/bin/bash -e + +# TODO: Review and if possible fix shellcheck errors. +# shellcheck disable=all + MODULES_VERSION=5.3.1 MODULES_BUILD_DIR=${BUILD_LOCATION} diff --git a/scripts/stage0/install_stage0.sh b/scripts/stage0/install_stage0.sh index 3cfde96..c2971cd 100644 --- a/scripts/stage0/install_stage0.sh +++ b/scripts/stage0/install_stage0.sh @@ -3,8 +3,9 @@ # TODO: Review and if possible fix shellcheck errors. # shellcheck disable=all -./scripts/stage0/install_env-modules.sh ./scripts/stage0/install_gcc.sh +./scripts/stage0/install_beegfs.sh +./scripts/stage0/install_modules.sh ./scripts/stage0/install_binutils.sh ./scripts/stage0/install_llvm.sh ./scripts/stage0/install_intel.sh