Skip to content
This repository was archived by the owner on Aug 5, 2022. It is now read-only.

Commit a6e1161

Browse files
fzou1fzou1
committed
enhance script of installing deps for multinode; fix issues in run_intelcaffe.sh
1 parent 53dae93 commit a6e1161

File tree

3 files changed

+204
-75
lines changed

3 files changed

+204
-75
lines changed

‎scripts/prepare_env.sh‎

Lines changed: 156 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,75 +1,198 @@
11
#!/bin/bash
22

3+
function usage
4+
{
5+
script_name=$0
6+
echo "Usage:"
7+
echo " $script_name [--host host_file]"
8+
echo ""
9+
echo " Parameters:"
10+
echo " host: host file includes list of nodes. Only used when you want to install dependencies for multinode"
11+
}
12+
13+
function check_os
14+
{
15+
# echo "Check OS and the version..."
16+
echo "Only CentOS is supported."
17+
}
18+
19+
function check_dependency
20+
{
21+
dep=$1
22+
which $dep >/dev/null 2>&1
23+
if [ $? -ne 0 ]; then
24+
echo "Warning: cannot find $dep"
25+
return 1
26+
fi
27+
return 0
28+
}
29+
30+
31+
sudo_passwd=""
32+
33+
function is_sudoer
34+
{
35+
echo $sudo_passwd | sudo -S -E -v >/dev/null
36+
if [ $? -eq 1 ]; then
37+
echo "User $(whoami) is not sudoer, and cannot install dependencies."
38+
exit 1
39+
fi
40+
}
41+
42+
# centos: yum; ubuntu: apt-get
343
os="centos"
44+
install_command=""
445

46+
check_os
47+
if [ "$os" == "centos" ]; then
48+
install_command="yum"
49+
check_dependency $install_command
50+
if [ $? -ne 0 ]; then
51+
echo "Please check if CentOS and $install_command is installed correctly."
52+
exit 1
53+
fi
54+
fi
55+
56+
package_installer="$install_command -y "
557
username=`whoami`
658
if [ "$username" != "root" ];
759
then
8-
package_installer="sudo -E"
60+
read -s -p "Enter password for $username: " sudo_passwd
61+
is_sudoer
62+
package_installer="echo $sudo_passwd | sudo -S -E $install_command -y "
963
fi
1064

11-
# centos: yum; ubuntu: apt-get
12-
package_installer+=" yum -y"
65+
1366

1467
function install_deps
1568
{
1669
echo "Install dependencies..."
1770
if [ "$os" == "centos" ]; then
18-
$package_installer clean all
19-
$package_installer upgrade
20-
$package_installer install epel-release
21-
$package_installer groupinstall "Development Tools"
71+
eval $package_installer clean all
72+
eval $package_installer upgrade
73+
eval $package_installer install epel-release
74+
eval $package_installer groupinstall "Development Tools"
2275
fi
23-
24-
$package_installer install python-devel boost boost-devel cmake numpy \
76+
77+
eval $package_installer install python-devel boost boost-devel cmake numpy \
2578
numpy-devel gflags gflags-devel glog glog-devel protobuf protobuf-devel hdf5 \
2679
hdf5-devel lmdb lmdb-devel leveldb leveldb-devel snappy-devel opencv \
2780
opencv-devel wget bc numactl
2881
}
2982

30-
function check_os
83+
function install_deps_multinode
3184
{
32-
echo "Check OS and the version..."
33-
}
85+
host_file=$1
86+
host_list=(`cat $host_file | sort | uniq`)
3487

88+
host_cnt=${#host_list[@]}
89+
if [ $host_cnt -eq 0 ]; then
90+
echo "Error: empty host list. Exit."
91+
exit 1
92+
fi
3593

36-
function checkout_source
37-
{
38-
echo "Checkout source code of Intel Caffe..."
39-
git clone https://github.com/intel/caffe.git
40-
if [ $? -eq 128 ]; then
41-
echo "Error during checking out source code. Please set proxy as below:"
42-
echo " export https_proxy=https://username:[email protected]:port"
94+
echo "Make sure you're executing command on host ${host_list[0]}"
95+
96+
echo $sudo_passwd | sudo -S -E yum -y clean all
97+
98+
if [ "$os" == "centos" ]; then
99+
eval $package_installer upgrade
100+
eval $package_installer install epel-release
101+
eval $package_installer clean all
102+
eval $package_installer groupinstall "Development Tools"
43103
fi
104+
105+
eval $package_installer install ansible
106+
107+
tmp_host_file=ansible_hosts.tmp
108+
ansible_host_file=/etc/ansible/hosts
109+
echo -e "[ourmaster]\n${host_list[0]}\n[ourcluster]\n" >$tmp_host_file
110+
for ((i=1; i<${#host_list[@]}; i++))
111+
do
112+
echo -e "${host_list[$i]}\n" >>$tmp_host_file
113+
done
114+
$command_prefix mv -f $tmp_host_file $ansible_host_file
115+
116+
ssh-keygen -t rsa -q
117+
for host in ${host_list[@]}
118+
do
119+
ssh-copy-id -i ~/.ssh/id_rsa.pub $host
120+
done
121+
ansible ourcluster -m ping
122+
123+
ansible all -m shell -a "$package_installer install python-devel boost boost-devel cmake numpy numpy-devel gflags gflags-devel glog glog-devel protobuf protobuf-devel hdf5 hdf5-devel lmdb lmdb-devel leveldb leveldb-devel snappy-devel opencv opencv-devel"
124+
125+
ansible all -m shell -a "$package_installer install mc cpuinfo htop tmux screen iftop iperf vim wget bc numactl"
126+
ansible all -m shell -a "systemctl stop firewalld.service"
44127
}
128+
45129
function build_caffe
46130
{
131+
is_multinode_=$1
132+
47133
echo "Build Intel Caffe..."
48134
cp Makefile.config.example Makefile.config
135+
136+
if [ $is_multinode_ -eq 1 ]; then
137+
echo "USE_MLSL := 1" >> Makefile.config
138+
echo "CAFFE_PER_LAYER_TIMINGS := 1" >> Makefile.config
139+
140+
mlslvars_sh=`find external/mlsl/ -name mlslvars.sh`
141+
if [ -f $mlslvars_sh ]; then
142+
source $mlslvars_sh
143+
fi
144+
fi
145+
49146
make -j 8
50147
}
51148

52-
function is_sudoer
149+
function sync_caffe_dir
53150
{
54-
sudo -v >/dev/null
55-
if [ $? -eq 1 ]; then
56-
echo "User $(whoami) is not sudoer, and cannot install dependencies."
57-
return 1
58-
fi
59-
return 0
151+
caffe_dir=`pwd`
152+
caffe_parent_dir=`dirname $caffe_dir`
153+
ansible ourcluster -m synchronize -a "src=$caffe_dir dest=$caffe_parent_dir"
60154
}
61155

62-
check_os
63-
if [ "$os" == "ubuntu" ]; then
64-
package_installer="apt-get"
65-
fi
66156

67-
is_sudoer
157+
host_file=""
158+
while [[ $# -gt 1 ]]
159+
do
160+
key="$1"
161+
case $key in
162+
--host)
163+
host_file="$2"
164+
shift
165+
;;
166+
--help)
167+
usage
168+
exit 0
169+
;;
170+
*)
171+
echo "Unknown option: $key"
172+
usage
173+
exit 1
174+
;;
175+
esac
176+
shift
177+
done
178+
179+
68180
if [ $? -eq 0 ]; then
69-
install_deps
181+
if [ "$host_file" == "" ]; then
182+
install_deps
183+
else
184+
install_deps_multinode $host_file
185+
fi
70186
fi
71187

188+
is_multinode=0
189+
if [ "$host_file" != "" ]; then
190+
is_multinode=1
191+
fi
192+
build_caffe $is_multinode
72193

73-
build_caffe
194+
if [ $is_multinode -eq 1 ]; then
195+
sync_caffe_dir
196+
fi
74197

75198
echo "Done."

‎scripts/run_benchmark.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/sh
1+
#!/bin/bash
22

33
# model path
44
model_path="models/intel_optimized_models"

0 commit comments

Comments
 (0)