diff --git a/Makefile b/Makefile
index 9ea6866..200b81c 100644
--- a/Makefile
+++ b/Makefile
@@ -188,27 +188,35 @@ else
   endif
 endif
 
-# Check if OpenMPI and NCCL are available, include them if so, for multi-GPU training
+# Check if NCCL is available, include if so, for multi-GPU training
 ifeq ($(NO_MULTI_GPU), 1)
-  $(info → Multi-GPU (OpenMPI + NCCL) is manually disabled)
+  $(info → Multi-GPU (NCCL) is manually disabled)
 else
   ifneq ($(OS), Windows_NT)
     # Detect if running on macOS or Linux
     ifeq ($(SHELL_UNAME), Darwin)
-      $(info ✗ Multi-GPU on CUDA on Darwin is not supported, skipping OpenMPI + NCCL support)
-    else ifeq ($(shell [ -d /usr/lib/x86_64-linux-gnu/openmpi/lib/ ] && [ -d /usr/lib/x86_64-linux-gnu/openmpi/include/ ] && echo "exists"), exists)
-      $(info ✓ OpenMPI found, OK to train with multiple GPUs)
-      NVCC_INCLUDES += -I/usr/lib/x86_64-linux-gnu/openmpi/include
-      NVCC_LDFLAGS += -L/usr/lib/x86_64-linux-gnu/openmpi/lib/
-      NVCC_LDLIBS += -lmpi -lnccl
+      $(info ✗ Multi-GPU on CUDA on Darwin is not supported, skipping NCCL support)
+    else ifeq ($(shell dpkg -l | grep -q nccl && echo "exists"), exists)
+      $(info ✓ NCCL found, OK to train with multiple GPUs)
+      NVCC_LDLIBS += -lnccl
       NVCC_FLAGS += -DMULTI_GPU
     else
-      $(info ✗ OpenMPI is not found, disabling multi-GPU support)
-      $(info ---> On Linux you can try install OpenMPI with `sudo apt install openmpi-bin openmpi-doc libopenmpi-dev`)
+      $(info ✗ NCCL is not found, disabling multi-GPU support)
+      $(info ---> On Linux you can try install NCCL with `sudo apt install libnccl2 libnccl-dev`)
     endif
   endif
 endif
 
+ifeq ($(USE_MPI), 1)
+  $(info → MPI is manually enabled)
+  NVCC_INCLUDES += -I/usr/lib/x86_64-linux-gnu/openmpi/include
+  NVCC_LDFLAGS += -L/usr/lib/x86_64-linux-gnu/openmpi/lib/
+  NVCC_FLAGS += -DUSE_MPI
+  NVCC_LDLIBS += -lmpi
+else
+  $(info → MPI is manually disabled)
+endif
+
 # Precision settings, default to bf16 but ability to override
 PRECISION ?= BF16
 VALID_PRECISIONS := FP32 FP16 BF16
diff --git a/llmc/zero.cuh b/llmc/zero.cuh
index dc7ba69..e85e6b8 100644
--- a/llmc/zero.cuh
+++ b/llmc/zero.cuh
@@ -198,6 +198,12 @@ ncclUniqueId get_nccl_id_via_fs(MultiGpuConfig* result, char* fs_path) {
     static char filename[1024];
     snprintf(filename, sizeof(filename), "%s/ncclUniqueId.sync", fs_path);
 
+    if (result->process_rank != 0) {
+        // Wait for the server to write the file
+        // This is a naive way to synchronize the processes
+        sleep(2);
+    }
+
     if (result->process_rank == 0) {
         ncclCheck(ncclGetUniqueId(&nccl_id));
         idFile = fopen(filename, "wb");
@@ -207,7 +213,7 @@ ncclUniqueId get_nccl_id_via_fs(MultiGpuConfig* result, char* fs_path) {
     } else {
         // Other ranks wait until the file is available and read the unique ID
         do {
-            usleep(1000000);  // 1 second
+            sleep(1);  // 1 second
             idFile = fopen(filename, "rb");
             if (idFile != NULL) break;
         } while (idFile == NULL);
diff --git a/scripts/multi_node/run_gpt2_124M_fs.sbatch b/scripts/multi_node/run_gpt2_124M_fs.sbatch
new file mode 100755
index 0000000..2498155
--- /dev/null
+++ b/scripts/multi_node/run_gpt2_124M_fs.sbatch
@@ -0,0 +1,84 @@
+#!/bin/bash
+#SBATCH --job-name=llmc-multinode                                     # job name
+#SBATCH --output=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.log   # output file
+#SBATCH --error=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.err    # error file
+#SBATCH --partition=llmc                                              # Specify the GPU partition
+#SBATCH --ntasks=16                                                   # total number of processes to launch on all nodes
+#SBATCH --nodes=2                                                     # total number of nodes
+#SBATCH --ntasks-per-node=8                                           # assuming each node has 8 gpus
+#SBATCH --gres=gpu:8                                                  # request 8 gpus from each node
+
+# NOTE: change the above slurm arguments to match your system!
+# Run with `sbatch <path_to_this_script.sh>`
+
+make train_gpt2cu USE_CUDNN=1 USE_MPI=0
+
+# NOTE: change the following to match your system
+binary_path="/home/ubuntu/llm.c/train_gpt2cu"
+out_dir="/ephemeral/data/fineweb/log_gpt2_124M_multi"
+train_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_train_*.bin'
+val_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_val_*.bin'
+sync_fs_path=$out_dir  # needs to be a shared filesystem path that all nodes can access
+
+# In case the file system is shared this is a no-op.
+# Otherwise, we need to copy the binary to all nodes.
+current_user=$USER
+hosts=$(scontrol show hostnames $SLURM_JOB_NODELIST)  # get the hostnames of the allocated nodes
+current_host=$(hostname)
+for host in $hosts; do
+    if [ $host == $current_host ]; then
+        continue
+    fi
+    echo "copying $binary_path to $current_user@$host"
+    scp -r $binary_path $current_user@$host:$binary_path
+done
+
+# Use this for NCCL debugging if you run into issues
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=ALL
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+# Optimization flags
+export NCCL_NET_GDR_LEVEL=2  # use GPUDirect RDMA - allows for direct memory access between GPUs across different nodes by bypassing the CPU
+export NCCL_IB_DISABLE=0  # use InfiniBand if available
+
+# NOTE: change the following environment variables to match your system - or comment them out if you don't need them
+export NCCL_SOCKET_IFNAME=ens17
+export OMPI_MCA_btl_tcp_if_include=ens17
+export NCCL_P2P_LEVEL=PXB
+
+if [ -z "$SLURM_JOB_ID" ]; then
+    echo "Make sure you're running in a SLURM environment. Did you forget to run with sbatch? Aborting."
+    exit 1
+else
+    DATESTRING=`date "+%Y-%m-%dT%H:%M:%S"`
+    echo "Running in a SLURM environment (job ID: $SLURM_JOB_ID, user: $current_user)"
+    echo "Running on hosts: $(echo $(scontrol show hostname))"
+    echo "$DATESTRING"
+fi
+
+srun -l -u bash -c "
+    $binary_path \
+    -i '$train_data_path' \
+    -j '$val_data_path' \
+    -o $out_dir \
+    -v 250 -s 20000 -g 144 \
+    -h 1 \
+    -b 64 -t 1024 \
+    -d 2097152 \
+    -r 0 \
+    -z 1 \
+    -c 0.1 \
+    -l 0.0006 \
+    -q 0.0 \
+    -u 700 \
+    -n 5000 \
+    -y 1 \
+    -e d12 \
+    -pn \$SLURM_NTASKS \
+    -pr \$SLURM_PROCID \
+    -pf $sync_fs_path \
+    -pi "fs" \
+"
+
+echo "$DATESTRING"
\ No newline at end of file
diff --git a/scripts/multi_node/run_gpt2_124M_mpi.sh b/scripts/multi_node/run_gpt2_124M_mpi.sh
new file mode 100755
index 0000000..366f8ff
--- /dev/null
+++ b/scripts/multi_node/run_gpt2_124M_mpi.sh
@@ -0,0 +1,49 @@
+
+make train_gpt2cu USE_CUDNN=1 USE_MPI=1
+
+# NOTE: change the following to match your system
+binary_path="/home/ubuntu/llm.c/train_gpt2cu"
+out_dir="/ephemeral/data/fineweb/log_gpt2_124M_multi"
+train_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_train_*.bin'
+val_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_val_*.bin'
+# You can find these names either in `/etc/hosts`` file or in the terminal (user@host:~$).
+host1="h100-node-1-0"  # master and worker node
+host2="h100-node-1-1"  # worker node
+
+# In case the file system is shared this is a no-op.
+# Otherwise, we need to copy the binary to all nodes.
+scp -r $binary_path $USER@$host2:$binary_path
+
+# Use this for NCCL debugging if you run into issues
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=ALL
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+# Optimization flags
+export NCCL_NET_GDR_LEVEL=2  # use GPUDirect RDMA - allows for direct memory access between GPUs across different nodes by bypassing the CPU
+export NCCL_IB_DISABLE=0  # use InfiniBand if available
+
+# NOTE: change the following environment variables to match your system - or comment them out if you don't need them
+export NCCL_SOCKET_IFNAME=ens17
+export OMPI_MCA_btl_tcp_if_include=ens17
+export NCCL_P2P_LEVEL=PXB
+
+mpirun -np 16 --host $host1:8,$host2:8 \
+    $binary_path \
+    -i "$train_data_path" \
+    -j "$val_data_path" \
+    -o $out_dir \
+    -v 250 -s 20000 -g 144 \
+    -h 1 \
+    -b 64 -t 1024 \
+    -d 2097152 \
+    -r 0 \
+    -z 1 \
+    -c 0.1 \
+    -l 0.0006 \
+    -q 0.1 \
+    -u 700 \
+    -n 1000 \
+    -y 0 \
+    -e d12 \
+    -pi "mpi" \
diff --git a/scripts/multi_node/run_gpt2_124M_tcp.sbatch b/scripts/multi_node/run_gpt2_124M_tcp.sbatch
new file mode 100755
index 0000000..fbf9c34
--- /dev/null
+++ b/scripts/multi_node/run_gpt2_124M_tcp.sbatch
@@ -0,0 +1,85 @@
+#!/bin/bash
+#SBATCH --job-name=llmc-multinode                                     # job name
+#SBATCH --output=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.log   # output file
+#SBATCH --error=/home/ubuntu/llm.c/scripts/multi_node/%x_%j_%t.err    # error file
+#SBATCH --partition=llmc                                              # Specify the GPU partition
+#SBATCH --ntasks=16                                                   # total number of processes to launch on all nodes
+#SBATCH --nodes=2                                                     # total number of nodes
+#SBATCH --ntasks-per-node=8                                           # assuming each node has 8 gpus
+#SBATCH --gres=gpu:8                                                  # request 8 gpus from each node
+
+# NOTE: change the above slurm arguments to match your system!
+# Run with `sbatch <path_to_this_script.sh>`
+
+make train_gpt2cu USE_CUDNN=1 USE_MPI=0
+
+# NOTE: change the following to match your system
+binary_path="/home/ubuntu/llm.c/train_gpt2cu"
+out_dir="/ephemeral/data/fineweb/log_gpt2_124M_multi"
+train_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_train_*.bin'
+val_data_path='/ephemeral/data/fineweb/bin_10B/fineweb_val_*.bin'
+# NOTE: change the server_ip to the IP address of the machine that is running process zero
+server_ip="10.0.1.220"
+
+# In case the file system is shared this is a no-op.
+# Otherwise, we need to copy the binary to all nodes.
+current_user=$USER
+hosts=$(scontrol show hostnames $SLURM_JOB_NODELIST)  # get the hostnames of the allocated nodes
+current_host=$(hostname)
+for host in $hosts; do
+    if [ $host == $current_host ]; then
+        continue
+    fi
+    echo "copying $binary_path to $current_user@$host"
+    scp -r $binary_path $current_user@$host:$binary_path
+done
+
+# Use this for NCCL debugging if you run into issues
+# export NCCL_DEBUG=INFO
+# export NCCL_DEBUG_SUBSYS=ALL
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+# Optimization flags
+export NCCL_NET_GDR_LEVEL=2  # use GPUDirect RDMA - allows for direct memory access between GPUs across different nodes by bypassing the CPU
+export NCCL_IB_DISABLE=0  # use InfiniBand if available
+
+# NOTE: change the following environment variables to match your system - or comment them out if you don't need them
+export NCCL_SOCKET_IFNAME=ens17
+export OMPI_MCA_btl_tcp_if_include=ens17
+export NCCL_P2P_LEVEL=PXB
+
+if [ -z "$SLURM_JOB_ID" ]; then
+    echo "Make sure you're running in a SLURM environment. Did you forget to run with sbatch? Aborting."
+    exit 1
+else
+    DATESTRING=`date "+%Y-%m-%dT%H:%M:%S"`
+    echo "Running in a SLURM environment (job ID: $SLURM_JOB_ID, user: $current_user)"
+    echo "Running on hosts: $(echo $(scontrol show hostname))"
+    echo "$DATESTRING"
+fi
+
+srun -l -u bash -c "
+    $binary_path \
+    -i '$train_data_path' \
+    -j '$val_data_path' \
+    -o $out_dir \
+    -v 250 -s 20000 -g 144 \
+    -h 1 \
+    -b 64 -t 1024 \
+    -d 2097152 \
+    -r 0 \
+    -z 1 \
+    -c 0.1 \
+    -l 0.0006 \
+    -q 0.0 \
+    -u 700 \
+    -n 5000 \
+    -y 1 \
+    -e d12 \
+    -pn \$SLURM_NTASKS \
+    -pr \$SLURM_PROCID \
+    -ps $server_ip \
+    -pi "tcp" \
+"
+
+echo "$DATESTRING"