Added files to create SMB share and basic scientific computing environment
This commit is contained in:
Adam Parler
parent
e4350fe8aa
commit
8945a8922f
2 changed files with 419 additions and 0 deletions
338
make_cluster.sh
Normal file
338
make_cluster.sh
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [[ "$BASH_SOURCE" == "$0" ]]; then
|
||||
echo "This script is the top-level bash file being run."
|
||||
echo "Setting up the environment for the cluster."
|
||||
BUILD_LOCATION=/avfs/backend
|
||||
INSTALL_LOCATION=/avfs/projects/ops
|
||||
mkdir -p ${BUILD_LOCATION}
|
||||
mkdir -p ${INSTALL_LOCATION}
|
||||
echo "Build location: ${BUILD_LOCATION}"
|
||||
echo "Install location: ${INSTALL_LOCATION}"
|
||||
else
|
||||
echo "This script is not the top-level bash file being run."
|
||||
fi
|
||||
|
||||
## INSTALL GCC FROM SOURCE
|
||||
GCC_BUILD_DIR=${BUILD_LOCATION}/gcc
|
||||
GCC_INSTALL_DIR=${INSTALL_LOCATION}/gcc
|
||||
mkdir -p ${GCC_INSTALL_DIR}
|
||||
|
||||
GCC_VERSION=7.5.0
|
||||
cd ${GCC_BUILD_DIR}
|
||||
mkdir gcc-${GCC_VERSION}-build
|
||||
|
||||
wget https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.gz
|
||||
tar -xf gcc-${GCC_VERSION}.tar.gz
|
||||
mv gcc-${GCC_VERSION} gcc-${GCC_VERSION}-source
|
||||
cd gcc-${GCC_VERSION}-source
|
||||
./contrib/download_prerequisites
|
||||
cd ../gcc-${GCC_VERSION}-build
|
||||
${GCC_BUILD_DIR}/gcc-${GCC_VERSION}-source/configure --prefix=${GCC_INSTALL_DIR}/${GCC_VERSION} --enable-languages=c,c++,fortran,go --disable-multilib --enable-bootstrap --enable-lto --enable-plugins
|
||||
make -j$(nproc)
|
||||
make install
|
||||
|
||||
## INSTALL ENVIRONMENT MODULES FROM SOURCE
|
||||
MODULES_BUILD_DIR=${BUILD_LOCATION}
|
||||
MODULES_INSTALL_DIR=${INSTALL_LOCATION}/modules
|
||||
mkdir -p ${MODULES_INSTALL_DIR}
|
||||
|
||||
cd ${MODULES_BUILD_DIR}
|
||||
MODULES_VERSION=5.3.1
|
||||
|
||||
curl -LJO https://github.com/cea-hpc/modules/releases/download/v${MODULES_VERSION}/modules-${MODULES_VERSION}.tar.gz
|
||||
tar xfz modules-${MODULES_VERSION}.tar.gz
|
||||
cd modules-${MODULES_VERSION}
|
||||
|
||||
mkdir build && cd build
|
||||
./configure --prefix=${MODULES_INSTALL_DIR}/Modules/${MODULES_VERSION} --modulefilesdir=${MODULES_INSTALL_DIR}/Modules/${MODULES_VERSION}/modulefiles
|
||||
make -j$(nproc)
|
||||
make install
|
||||
ln -s ${MODULES_INSTALL_DIR}/Modules/${MODULES_VERSION}/init/profile.sh /etc/profile.d/modules.sh
|
||||
ln -s ${MODULES_INSTALL_DIR}/Modules/${MODULES_VERSION}/init/profile.csh /etc/profile.d/modules.csh
|
||||
|
||||
## INSTALL OPENSSL FOR CMAKE
|
||||
SSL_BUILD_DIR=${BUILD_LOCATION}/openssl
|
||||
SSL_INSTALL_DIR=${INSTALL_LOCATION}/openssl
|
||||
mkdir -p ${SSL_BUILD_DIR}
|
||||
mkdir -p ${SSL_INSTALL_DIR}
|
||||
|
||||
SSL_VERSION=1.1.1w
|
||||
|
||||
cd ${SSL_BUILD_DIR}
|
||||
wget -c https://www.openssl.org/source/openssl-${SSL_VERSION}.tar.gz
|
||||
tar -xf openssl-${SSL_VERSION}.tar.gz
|
||||
cd openssl-${SSL_VERSION}
|
||||
mkdir build && cd build
|
||||
../config --prefix=${SSL_INSTALL_DIR}/${SSL_VERSION} --openssldir=${SSL_INSTALL_DIR}/${SSL_VERSION}
|
||||
make -j$(nproc)
|
||||
make -j$(nproc) test
|
||||
make install
|
||||
|
||||
## INSTALL Qt FOR CMAKE-GUI
|
||||
QT_BUILD_DIR=${BUILD_LOCATION}/qt
|
||||
QT_INSTALL_DIR=${INSTALL_LOCATION}/qt
|
||||
mkdir -p ${QT_BUILD_DIR}
|
||||
mkdir -p ${QT_INSTALL_DIR}
|
||||
|
||||
## SOME COMBINATION WILL INSTALL GL
|
||||
install build-essential libgl1-mesa-dev libglu1-mesa-dev libglut-dev libgl2-mesa-dev libdrm-dev
|
||||
|
||||
QT_VERSION=5.15.12
|
||||
QT_MAJOR_VERSION=$(echo ${QT_VERSION} | cut -d. -f1-2)
|
||||
|
||||
cd ${QT_BUILD_DIR}
|
||||
wget -c https://download.qt.io/official_releases/qt/${QT_MAJOR_VERSION}/${QT_VERSION}/single/qt-everywhere-opensource-src-${QT_VERSION}.tar.xz
|
||||
tar -xf qt-everywhere-opensource-src-${QT_VERSION}.tar.xz
|
||||
cd qt-everywhere-opensource-src-${QT_VERSION}
|
||||
mkdir build && cd build
|
||||
../configure -prefix ${QT_INSTALL_DIR}/${QT_VERSION} -make tests -release -opensource -confirm-license
|
||||
|
||||
## INSTALL CMAKE FROM SOURCE
|
||||
CMAKE_BUILD_DIR=${BUILD_LOCATION}/cmake
|
||||
CMAKE_INSTALL_DIR=${INSTALL_LOCATION}/cmake
|
||||
mkdir -p ${CMAKE_BUILD_DIR}
|
||||
mkdir -p ${CMAKE_INSTALL_DIR}
|
||||
|
||||
VERSIONS=(3.5.2
|
||||
3.12.4
|
||||
3.17.5
|
||||
3.21.3
|
||||
3.24.4
|
||||
3.28.1)
|
||||
|
||||
OPENSSL_ROOT_DIR=${SSL_INSTALL_DIR}/${SSL_VERSION}
|
||||
for CMAKE_VERSION in "${strings[@]}"; do
|
||||
# CMAKE_VERSION=3.21.3
|
||||
CMAKE_MAJOR_VERSION=$(echo ${CMAKE_VERSION} | cut -d. -f1-2)
|
||||
cd ${CMAKE_BUILD_DIR}
|
||||
wget -c http://www.cmake.org/files/v${CMAKE_MAJOR_VERSION}/cmake-${CMAKE_VERSION}.tar.gz
|
||||
tar -xf cmake-${CMAKE_VERSION}.tar.gz
|
||||
mkdir -p cmake-${CMAKE_VERSION}/build && cd cmake-${CMAKE_VERSION}/build
|
||||
|
||||
../configure --prefix=${CMAKE_INSTALL_DIR}/${CMAKE_VERSION} --parallel=$(nproc)
|
||||
make -j$(nproc)
|
||||
make install
|
||||
done
|
||||
|
||||
## INSTALL GIT FROM SOURCE
|
||||
GIT_BUILD_DIR=${BUILD_LOCATION}/git
|
||||
GIT_INSTALL_DIR=${INSTALL_LOCATION}/git
|
||||
mkdir -p ${GIT_BUILD_DIR}
|
||||
mkdir -p ${GIT_INSTALL_DIR}
|
||||
|
||||
cd ${GIT_BUILD_DIR}
|
||||
GIT_VERSION=2.11.3
|
||||
wget -c https://mirrors.edge.kernel.org/pub/software/scm/git/git-${GIT_VERSION}.tar.gz
|
||||
tar -xzf git-${GIT_VERSION}.tar.gz
|
||||
cd git-${GIT_VERSION}
|
||||
make configure
|
||||
./configure --prefix=${GIT_INSTALL_DIR}/${GIT_VERSION}
|
||||
make -j$(nproc)
|
||||
make test
|
||||
make install
|
||||
|
||||
## INSTALL LLVM FROM SOURCE
|
||||
LLVM_BUILD_DIR=${BUILD_LOCATION}/llvm
|
||||
LLVM_INSTALL_DIR=${INSTALL_LOCATION}/llvm
|
||||
mkdir -p ${LLVM_BUILD_DIR}
|
||||
mkdir -p ${LLVM_INSTALL_DIR}
|
||||
|
||||
LLVM_VERSION=13.0.0
|
||||
|
||||
git clone https://github.com/llvm/llvm-project.git ${LLVM_BUILD_DIR}/llvm-${LLVM_VERSION}
|
||||
cd ${LLVM_BUILD_DIR}/llvm-${LLVM_VERSION}
|
||||
git checkout llvmorg-${LLVM_VERSION}
|
||||
mkdir build && cd build
|
||||
cmake -S llvm -B build -G "Unix Makefiles" \
|
||||
-DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb;openmp;parallel-libs;polly" \
|
||||
-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${LLVM_INSTALL_DIR}/${LLVM_VERSION} \
|
||||
-DLLVM_INSTALL_UTILS=ON llvm
|
||||
make -j$(nproc)
|
||||
make install
|
||||
|
||||
module load gcc/${GCC_VERSION} cmake/${CMAKE_VERSION}
|
||||
|
||||
## INSTALL OPENMPI FROM SOURCE
|
||||
|
||||
if [ $(command -v srun) ]; then
|
||||
echo "Slurm installation found. OpenMPI will be configured with --with-pmi."
|
||||
EXTRA_CONFIGURE_FLAGS="--with-pmi"
|
||||
else
|
||||
EXTRA_CONFIGURE_FLAGS=""
|
||||
fi
|
||||
|
||||
./configure CFLAGS="${CFLAGS}" \
|
||||
--prefix=${pkg_install_dir} \
|
||||
--libdir="${pkg_install_dir}/lib" \
|
||||
--enable-mpi1-compatibility \
|
||||
${EXTRA_CONFIGURE_FLAGS}
|
||||
|
||||
## INSTALL MPICH FROM SOURCE
|
||||
|
||||
|
||||
./configure \
|
||||
--prefix="${pkg_install_dir}" \
|
||||
MPICC="" \
|
||||
FFLAGS="${FCFLAGS} ${compat_flag}" \
|
||||
FCFLAGS="${FCFLAGS} ${compat_flag}" \
|
||||
--without-x \
|
||||
--enable-gl=no \
|
||||
--with-device=${MPICH_DEVICE} \
|
||||
|
||||
make -j$(nproc)
|
||||
make install
|
||||
|
||||
|
||||
MPIRUN="${pkg_install_dir}/bin/mpiexec"
|
||||
MPICC="${pkg_install_dir}/bin/mpicc"
|
||||
MPICXX="${pkg_install_dir}/bin/mpicxx"
|
||||
MPIFC="${pkg_install_dir}/bin/mpifort"
|
||||
MPIFORT="${MPIFC}"
|
||||
MPIF77="${MPIFC}"
|
||||
MPICH_CFLAGS="-I'${pkg_install_dir}/include'"
|
||||
MPICH_LDFLAGS="-L'${pkg_install_dir}/lib' -Wl,-rpath,'${pkg_install_dir}/lib'"
|
||||
|
||||
export MPI_MODE="${MPI_MODE}"
|
||||
export MPIRUN="${MPIRUN}"
|
||||
export MPICC="${MPICC}"
|
||||
export MPICXX="${MPICXX}"
|
||||
export MPIFC="${MPIFC}"
|
||||
export MPIFORT="${MPIFORT}"
|
||||
export MPIF77="${MPIF77}"
|
||||
export MPICH_CFLAGS="${MPICH_CFLAGS}"
|
||||
export MPICH_LDFLAGS="${MPICH_LDFLAGS}"
|
||||
export MPICH_LIBS="${MPICH_LIBS}"
|
||||
export MPI_CFLAGS="${MPICH_CFLAGS}"
|
||||
export MPI_LDFLAGS="${MPICH_LDFLAGS}"
|
||||
export MPI_LIBS="${MPICH_LIBS}"
|
||||
export CP_DFLAGS="\${CP_DFLAGS} IF_MPI(-D__parallel|)"
|
||||
export CP_CFLAGS="\${CP_CFLAGS} IF_MPI(${MPICH_CFLAGS}|)"
|
||||
export CP_LDFLAGS="\${CP_LDFLAGS} IF_MPI(${MPICH_LDFLAGS}|)"
|
||||
export CP_LIBS="\${CP_LIBS} IF_MPI(${MPICH_LIBS}|)"
|
||||
|
||||
## INSTALL OPENBLAS FROM SOURCE
|
||||
OPENBLAS_BUILD_DIR=${BUILD_LOCATION}/openblas
|
||||
OPENBLAS_INSTALL_DIR=${INSTALL_LOCATION}/openblas
|
||||
mkdir -p ${OPENBLAS_BUILD_DIR}
|
||||
mkdir -p ${OPENBLAS_INSTALL_DIR}
|
||||
|
||||
OPENBLAS_VERSION=0.3.18
|
||||
|
||||
cd ${OPENBLAS_BUILD_DIR}
|
||||
wget -c openblas
|
||||
tar -xf openblas-${OPENBLAS_VERSION}.tar.gz
|
||||
cd openblas-${OPENBLAS_VERSION}
|
||||
make -j$(nproc)
|
||||
make install PREFIX=${OPENBLAS_INSTALL_DIR}/${OPENBLAS_VERSION}
|
||||
|
||||
## INSTALL LAPACK FROM SOURCE
|
||||
LAPACK_BUILD_DIR=${BUILD_LOCATION}/lapack
|
||||
LAPACK_INSTALL_DIR=${INSTALL_LOCATION}/lapack
|
||||
mkdir -p ${LAPACK_BUILD_DIR}
|
||||
mkdir -p ${LAPACK_INSTALL_DIR}
|
||||
|
||||
LAPACK_VERSION=3.10.0
|
||||
|
||||
## INSTALL HDF5 FROM SOURCE
|
||||
module load gcc/8.1.0 openmpi/4.1.4 cmake/3.21.3
|
||||
|
||||
# Check if MPI is enabled
|
||||
if [ -n "$MPI_ENABLED" ]; then
|
||||
MPI_ENABLED=true
|
||||
else
|
||||
MPI_ENABLED=false
|
||||
fi
|
||||
|
||||
HDF5_BUILD_DIR=${BUILD_LOCATION}/hdf5
|
||||
HDF5_VERSION=1.12.1
|
||||
|
||||
# Use the MPI_ENABLED variable in your script
|
||||
if [ "$MPI_ENABLED" = true ]; then
|
||||
# MPI is enabled, do something
|
||||
echo "MPI is enabled"
|
||||
HDF5_INSTALL_DIR=${INSTALL_LOCATION}/hdf5-withmpi/${HDF5_VERSION}/${COMPILER}/${COMPILER_VERSION}/${MPI_NAME}/${MPI_VERSION}
|
||||
else
|
||||
# MPI is not enabled, do something else
|
||||
echo "MPI is not enabled"
|
||||
HDF5_INSTALL_DIR=${INSTALL_LOCATION}/hdf5/${HDF5_VERSION}/${COMPILER}/${COMPILER_VERSION}
|
||||
fi
|
||||
|
||||
## INSTALL NETCDF FROM SOURCE
|
||||
|
||||
## INSTALL OPENMC FROM SOURCE
|
||||
module load gcc/8.1.0 cmake/3.21.3 python/3.11.5 git/2.11.3
|
||||
|
||||
OPENMC_BUILD_DIR=${BUILD_LOCATION}/openmc
|
||||
OPENMC_INSTALL_DIR=${INSTALL_LOCATION}/openmc
|
||||
|
||||
mkdir -p ${OPENMC_BUILD_DIR}
|
||||
mkdir -p ${OPENMC_INSTALL_DIR}
|
||||
|
||||
PROJECT_FOLDER=/avfs/projects/openmc
|
||||
cd ${PROJECT_FOLDER}
|
||||
python3 -m venv .venv
|
||||
|
||||
OPENMC_VERSION=0.14.0
|
||||
|
||||
cd ${OPENMC_BUILD_DIR}
|
||||
git clone --recurse-submodules https://github.com/openmc-dev/openmc.git
|
||||
cd openmc
|
||||
git checkout v${OPENMC_VERSION}
|
||||
mkdir build && cd build
|
||||
|
||||
if [ "$MPI_ENABLED" = true ]; then
|
||||
# MPI is enabled, do something
|
||||
# HDF5_ROOT=/avfs/projects/hdf5-withmpi/gcc/8.1.0/openmpi/4.1.4 \
|
||||
# CC=/avfs/projects/openmpi/4.1.4/gcc/8.1.0/bin/mpicc \
|
||||
# CXX=/avfs/projects/openmpi/4.1.4/gcc/8.1.0/bin/mpicxx \
|
||||
module load hdf5-withmpi openmpi/4.1.4
|
||||
HDF5_ROOT=${HDF5_ROOT} CC=mpicc CXX=mpicxx \
|
||||
cmake .. -DCMAKE_INSTALL_PREFIX=${OPENMC_INSTALL_DIR}/${OPENMC_VERSION} -DOPENMC_USE_MPI=ON -DOPENMC_USE_OPENMP=ON -DCMAKE_BUILD_TYPE=Release
|
||||
else
|
||||
# MPI is not enabled, do something else
|
||||
module load hdf5
|
||||
HDF5_ROOT=${HDF5_ROOT}
|
||||
cmake .. -DCMAKE_INSTALL_PREFIX=${OPENMC_INSTALL_DIR}/${OPENMC_VERSION} -DOPENMC_USE_MPI=OFF -DOPENMC_USE_OPENMP=ON -DCMAKE_BUILD_TYPE=Release
|
||||
fi
|
||||
|
||||
make -j20
|
||||
make install
|
||||
cd ..
|
||||
source ${PROJECT_FOLDER}/.venv/bin/activate
|
||||
|
||||
if [ "$MPI_ENABLED" = true ]; then
|
||||
# MPI is enabled, do something
|
||||
MPICC=/avfs/projects/openmpi/4.1.4/gcc/8.1.0/bin/mpicc \
|
||||
/avfs/projects/python/3.11.5/bin/python3 -m pip install mpi4py
|
||||
CC=/avfs/projects/openmpi/4.1.4/gcc/8.1.0/bin/mpicc HDF5_MPI=ON \
|
||||
HDF5_DIR=/avfs/projects/hdf5-withmpi/gcc/8.1.0/openmpi/4.1.4 \
|
||||
${PROJECT_FOLDER}/.venv/bin/python3 -m pip install --no-binary=h5py h5py
|
||||
${PROJECT_FOLDER}/.venv/bin/python3 -m pip install .
|
||||
else
|
||||
# MPI is not enabled, do something else
|
||||
HDF5_DIR=${HDF5_ROOT}
|
||||
${PROJECT_FOLDER}/.venv/bin/python3 -m pip install .
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# HDF5_ROOT=/share/apps/hdf5-withmpi/gcc/8.1.0/openmpi/4.1.4 CC=/share/apps/openmpi/4.1.4/gcc/8.1.0/bin/mpicc CXX=/share/apps/openmpi/4.1.4/gcc/8.1.0/bin/mpicxx cmake .. -DCMAKE_INSTALL_PREFIX=/qfs/people/parl703/openmc -DOPENMC_USE_MPI=ON -DOPENMC_USE_OPENMP=ON -DCMAKE_BUILD_TYPE=Release
|
||||
# make -j20
|
||||
# make install
|
||||
# cd ..
|
||||
# MPICC=/share/apps/openmpi/4.1.4/gcc/8.1.0/bin/mpicc /share/apps/python/3.11.5/bin/python3 -m pip install mpi4py
|
||||
# CC=/share/apps/openmpi/4.1.4/gcc/8.1.0/bin/mpicc HDF5_MPI=ON HDF5_DIR=/share/apps/hdf5-withmpi/gcc/8.1.0/openmpi/4.1.4 /rcfs/projects/sherlock/openmc/dwnld/.venv/bin/python3 -m pip install --no-binary=h5py h5py
|
||||
# /rcfs/projects/sherlock/openmc/dwnld/.venv/bin/python3 -m pip install .
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
81
set_smb_share.sh
Normal file
81
set_smb_share.sh
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
SHARE_DIR=/data/share
|
||||
|
||||
## DEBIAN
|
||||
sudo apt-get install samba smbclient cifs-utils
|
||||
|
||||
## CENTOS
|
||||
sudo yum install samba samba-common samba-client
|
||||
|
||||
## Create a directory to share
|
||||
sudo mkdir -p ${SHARE_DIR}
|
||||
sudo chmod -R 2770 ${SHARE_DIR}
|
||||
sudo chown -R nobody:nogroup ${SHARE_DIR}
|
||||
sudo chcon -t samba_share_t ${SHARE_DIR}
|
||||
|
||||
cat << EOF >> /etc/samba/smb.conf.bkp
|
||||
[global]
|
||||
interfaces = 10.0.0.0/24 eth0
|
||||
# hosts allow = 127.0.0.1/8 192.168.1.0/24
|
||||
workgroup = WORKGROUP
|
||||
server string = Samba Server %v
|
||||
netbios name = MYSHARES
|
||||
# security = user
|
||||
unix password sync = Yes
|
||||
passwd program = /usr/bin/passwd %u
|
||||
map to guest = bad user
|
||||
dns proxy = no
|
||||
|
||||
[share]
|
||||
path = ${SHARE_DIR}
|
||||
available = yes
|
||||
valid users = ${USER}
|
||||
read only = no
|
||||
browsable = yes
|
||||
public = no
|
||||
writable = yes
|
||||
|
||||
[Public]
|
||||
comment = Public Folder
|
||||
path = ${SHARE_DIR}/public
|
||||
writable = yes
|
||||
guest ok = yes
|
||||
guest only = yes
|
||||
force create mode = 770
|
||||
force directory mode = 770
|
||||
|
||||
[Private]
|
||||
comment = Private Folder
|
||||
path = ${SHARE_DIR}
|
||||
writable = yes
|
||||
guest ok = no
|
||||
valid users = @smbshare
|
||||
# force create mode = 660
|
||||
# force directory mode = 770
|
||||
inherit permissions = yes
|
||||
|
||||
EOF
|
||||
|
||||
sudo testparm /etc/samba/smb.conf.bkp
|
||||
|
||||
## IF EXECUTION WORKS, THEN MOVE THE FILE TO THE ORIGINAL PATH
|
||||
sudo mv /etc/samba/smb.conf.bkp /etc/samba/smb.conf
|
||||
sudo useradd -M -s /sbin/nologin sambauser
|
||||
sudo usermod -aG smbshare sambauser
|
||||
sudo smbpasswd -a sambauser
|
||||
sudo smbpasswd -e sambauser
|
||||
|
||||
## DEBIAN
|
||||
sudo systemctl restart nmbd
|
||||
sudo systemctl restart smbd
|
||||
sudo ufw allow from 192.168.205.0/24 to any app Samba
|
||||
|
||||
## CENTOS
|
||||
sudo firewall-cmd --add-service=samba --zone=public --permanent
|
||||
sudo firewall-cmd --reload
|
||||
sudo systemctl start smb
|
||||
sudo systemctl enable smb
|
||||
sudo systemctl status smb
|
||||
|
||||
sudo systemctl start nmb
|
||||
sudo systemctl enable nmb
|
||||
sudo systemctl status nmb
|
||||
Loading…
Add table
Add a link
Reference in a new issue