Hypervisor

From DISI
(Redirected from Libvirt)
Jump to navigation Jump to search

ssh to hypervisor, run commands

sudo virsh
list --all
start (full name)
shutdown (number)

Configuration Steps

configure/test public network

edit ifcfg-eth1 - public network Note: This eth1 configuration is only for testing that access to public network operates correctly. It will not be used permanently. 

[root@aleph2 networks]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
DEVICE=eth1
BOOTPROTO=static
DHCP_HOSTNAME=aleph2.cluster.ucsf.bkslab.org
HWADDR=00:1E:0B:D5:43:B1
IPADDR=169.230.26.54
PREFIX="24"
GATEWAY="169.230.26.1"
NM_CONTROLLED=no
ONBOOT=yes
TYPE=Ethernet
UUID="993a14ad-4b0b-41a3-a15e-d29d7ee5e61c"

[root@aleph2 networks]# ping google.com
PING google.com (172.217.6.78) 56(84) bytes of data.
64 bytes from sfo07s17-in-f78.1e100.net (172.217.6.78): icmp_seq=1 ttl=52 time=3.66 ms
[root@aleph2 networks]# traceroute google.com
traceroute to google.com (172.217.6.78), 30 hops max, 60 byte packets
1  gw-26.compbio.ucsf.edu (169.230.26.1)  1.124 ms  1.136 ms  1.398 ms
2  3002-s251-r08.ucsf.edu (169.230.0.165)  0.326 ms  0.355 ms  0.411 ms
3  3002-s251-r06.ucsf.edu (169.230.251.46)  0.755 ms  0.826 ms  0.887 ms
4  2408-0c50-r05.ucsf.edu (128.218.41.58)  0.599 ms  0.643 ms  0.694 ms
5  2252-00f1-fa01 (128.218.41.26)  0.922 ms  0.876 ms  0.855 ms
6  2252-00f1-r01.ucsf.edu (128.218.41.9)  1.224 ms  1.214 ms  1.113 ms
7  sfo-agg1--ucsf-10ge.cenic.net (137.164.3.137)  1.460 ms  1.475 ms  1.459 ms
8  dc-svl-agg4--sfo-agg1-10ge-1.cenic.net (137.164.22.27)  3.330 ms  3.228 ms  3.313 ms
9  74.125.147.146 (74.125.147.146)  3.177 ms  3.287 ms  4.204 ms
10  108.170.242.81 (108.170.242.81)  3.653 ms  3.651 ms  3.639 ms
11  209.85.248.35 (209.85.248.35)  3.462 ms 209.85.247.55 (209.85.247.55)  3.367 ms  3.440 ms
12  sfo07s17-in-f14.1e100.net (172.217.6.78)  3.566 ms  3.431 ms  3.642 ms

create network bridges

ifcfg-br0 (private network) 

DEVICE="br0"
TYPE="Bridge"
ONBOOT="yes"
BOOTPROTO="dhcp"
DHCP_HOSTNAME="tet.cluster.ucsf.bkslab.org"
HOSTNAME="tet.cluster.ucsf.bkslab.org"
DOMAIN="cluster.ucsf.bkslab.org"
STP="off"
DELAY="0"

ifcfg-br1 - public network

DEVICE="br1"
TYPE="Bridge"
ONBOOT="yes"
NM_CONTROLLED="no"
BOOTPROTO="static"
IPADDR="169.230.26.189"
PREFIX="24"
GATEWAY="169.230.26.1"
STP="off"
DELAY="0"

libvirt to define networks

yum install libvirt
chkconfig --levels=345 libvirtd on
service libvirtd start

mkdir ~/libvirt
<get default files>
virsh net-define cluster-network.xml
virsh net-start cluster
virsh net-autostart cluster

virsh net-define public-network.xml
virsh net-start public
virsh net-autostart public

virsh pool-define default-pool.xml
virsh pool-start default
virsh pool-autostart default

echo "
vnc_tls = 1
vnc_tls_x509_verify = 1
vnc_tls_x509_cert_dir = "/etc/pki/libvirt-vnc"
vnc_listen = "<CLUSTER IP ADDRESS>"
vnc_password = "4nu6rho3"
user = "qemu"
group = "qemu"
" >> /etc/libvirtd/qemu.conf

echo "
log_level = 1
auth_tcp = "none"
log_outputs="3:syslog:libvirtd"
listen_tls = 1
listen_tcp = 0
" >> /etc/libvirtd/libvirtd.conf

echo 'LIBVIRTD_ARGS="--listen"' >> /etc/sysconfig/libvirtd


#below command wipes out iptables.  be careful.
lokkit -s libvirtd-tls

List of hypervisors

Hypervisor Name Cluster Rack # Hypervisor # Runs Machine
vmware1 0 1 1 mostly linux services DL165G5 ??
vmware2 0 1 2 windows, obsolete DL165G5 ??
apes 1 1 1 ?? 64 way C6145
rocky 1 1 6 lots... 64 way C6145
pulp 1 1 2 lots... 64 Way C6145
aleph 2 0 1 alpha,beta,gamma 8 way DL160g5
kaf 2 0 4 none SiM 4-in-1 32-way
vav 2 0 6 ntpd machine DL160G5

Set up hypervisor

This is aleph:/root/setup/libvirt/SETUP 

virsh pool-define default-pool.xml
virsh pool-define fast-pool.xml
virsh pool-start fast
virsh pool-autostart fast
virsh pool-define bulk-pool.xml
virsh pool-start bulk
virsh pool-autostart bulk

virsh net-define cluster-network.xml
virsh net-start cluster
virsh net-autostart cluster
virsh net-define public-network.xml
virsh net-start public
virsh net-autostart public

BKS SGE Cell

https://alpha.compbio.ucsf.edu/config_templates/43-todo-sge-execd/edit

set up first VM

These instructions are from /root/setup/vms/alpha-setup 

#!/bin/sh

#TODO:
# Resolv.conf
# Firewalls

CLUSTER_MAC=$( ip addr list br0 | tail -n+2 | head -n1 | tr -s ' ' | cut -d\  -f3 )
CLUSTER_IP=$( ip addr list br0 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f1 )
CLUSTER_PREFIX=$( ip addr list br0 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f2 )
PUBLIC_IP=$( ip addr list br1 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f1 )

sed -i "^$PUBLIC_IP " /etc/hosts
echo "$PUBLIC_IP     $( hostname )"  >> /etc/hosts

SHORT_HOSTNAME=$( hostname s )
PUBLIC_HOSTNAME=$( hostname )
PUBLIC_DOMAIN=$( hostname -d )
CLUSTER_DOMAIN=cluster.$PUBLIC_DOMAIN
CLUSTER_HOSTNAME=$SHORT_HOSTNAME.$CLUSTER_DOMAIN
CLUSTER_OCTETS=($( echo $CLUSTER_IP | tr . "\n" ))
CLUSTER_NETWORK=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.0.0
CLUSTER_MASK=$( ifconfig br0 | grep Mask | tr -d ' ' | cut -d: -f4 )
CLUSTER_START=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.0.1
CLUSTER_END=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.99.99

sed -i "^$CLUSTER_IP " /etc/hosts
sed -i "^$PUBLIC_IP " /etc/hosts
echo "$CLUSTER_IP    $CLUSTER_HOSTNAME" >> /etc/hosts
echo "$PUBLIC_IP     $PUBLIC_HOSTNAME"  >> /etc/hosts

ALPHA=10.20.1.1
EPEL=epel-release-6-8.noarch.rpm

echo " - Generating/Adding ssh-key (Enter password twice)"
ssh-copy-id $ALPHA
eval $( ssh-agent )
ssh-add ~/.ssh/id_rsa

echo " - Transfering repository setup files"
scp ../pkgs/* $ALPHA:~
ssh $ALPHA "yum -y install ~/epel-release-6-8.noarch.rpm"
ssh $ALPHA "mv -v ~/scl.repo /etc/yum.repos.d/scl.repo"
ssh $ALPHA "yum -y update"

echo " - Transfering service setup files"
scp ../services/foreman $ALPHA:~
ssh $ALPHA "sh ~/foreman"

echo " - Configuring Foreman Server"
# Allow: dns, tftp, http, https, nfs, puppet, dhcp
# TODO: network SSH
ssh $ALPHA "lokkit -s dns -s tftp -s http -s https -s nfs -p 8140:tcp -p 67:tcp -p 67:udp "
lokkit -s libvirt -s libvirt-tls

echo " - Hardcoding DNS entries"
ssh $ALPHA "grep alpha /etc/hosts" >> /etc/hostsa


echo " - Retrieving configuration credentials"
scp -r $ALPHA:~/.foreman ~

echo " - Installing local CLI tools"
yum -y install http://yum.theforeman.org/releases/latest/el6/x86_64/foreman-release.rpm
yum -y install rubygem-hammer_cli rubygem-hammer_cli_foreman


echo " - Finalizing Installation (Making Proxy Connections)"
CLUSTER_DOMAIN_ID=$( hammer domain info --name=$CLUSTER_DOMAIN | grep ^Id: | cut -d: -f2 | tr -d ' ' )


hammer host info --id=1
hammer -v host update --id=$CLUSTER_DOAMIN_ID
hammer proxy info --id=1
hammer -v domain update --name=$CLUSTER_DOMAIN --dns-id=1
hammer -v domain update --name=$PUBLIC_DOMAIN --dns-id=1
hammer -v subnet create \
	--name "$CLUSTER_DOMAIN" \
	--network "$CLUSTER_NETWORK" \
	--mask "$CLUSTER_MASK" \
	--dns-primary "$ALPHA" \
	--from "$CLUSTER_START" \
	--to "$CLUSTER_END" \
	--domain-ids "$CLUSTER_DOMAIN_ID" \
	--dhcp-id 1 \
	--dns-id 1 \
	--tftp-id 1

CLUSTER_SUBNET_ID=$( hammer subnet info --name=$CLUSTER_DOMAIN | grep ^Id: | cut -d: -f2 | tr -d ' ' )

echo " - Registering self with Foreman"
# OS is always 1
FOREMAN_OS=$( hammer os info --id 1 | grep ^Id: | cut -d: -f2 | tr -d ' ' )
FOREMAN_ARCH=$( hammer architecture info --name `uname -i` | grep ^Id: | cut -d: -f2 | tr -d ' ' )

hammer -v host create \
	--name "$CLUSTER_HOSTNAME" \
	--ip "$CLUSTER_IP" \
	--mac "$CLUSTER_MAC" \
        --domain-id "$CLUSTER_DOMAIN_ID" \
	--subnet-id "$CLUSTER_SUBNET_ID" \
	--architecture-id "$FOREMAN_ARCH" \
	--operatingsystem-id "$FOREMAN_OS" \
	--managed "true" \
	--build "false"

echo " - Registering with Puppet"
puppet agent --waitforcert 60 --no-daemonize &
ssh $ALPHA "puppet cert sign $( hostname )"
service puppet restart

echo " - Updating Certificates"
mkdir -pv /etc/pki/CA
ln -sv /var/lib/puppet/ssl/certs/ca.pem /etc/pki/CA/cacert.pem
rm -rf /etc/pki/libvirt

mkdir -pv /etc/pki/libvirt/private
cp -rv /var/lib/puppet/ssl/private_keys/`hostname`.pem /etc/pki/libvirt/private/serverkey.pem
cp -rv /var/lib/puppet/ssl/certs/`hostname`.pem /etc/pki/libvirt/servercert.pem 
ln -sv /etc/pki/libvirt/private/serverkey.pem /etc/pki/libvirt/private/clientkey.pem
ln -sv /etc/pki/libvirt/servercert.pem /etc/pki/libvirt/clientcert.pem
chcon -Rv -u system_u /etc/pki/libvirt
chown -Rv root:qemu /etc/pki/libvirt
chmod -Rv 440 /etc/pki/libvirt

mkdir -pv /etc/pki/libvirt-vnc
ln -sv /etc/pki/CA/cacert.pem /etc/pki/libvirt-vnc/ca-cert.pem
ln -sv /etc/pki/libvirt/servercert.pem /etc/pki/libvirt-vnc/server-cert.pem
ln -sv /etc/pki/libvirt/private/serverkey.pem /etc/pki/libvirt-vnc/server-key.pem

sed -i -e '/^listen_tls =/d' '/listen_tcp =/d' /etc/libvirt/libvirtd.conf
echo >> /etc/libvirt/libvirtd.conf << 'EOF'
listen_tls = 1
listen_tcp = 0
EOF

sed -i -e '/vnc_tls =/d' '/vnc_listen =/d' /etc/libvirt/qemu.conf
echo "
vnc_tls = 1
vnc_listen = $CLUSTER_IP
" >> /etc/libvirt/qemu.conf

service libvirtd restart

ssh $ALPHA << 'EOF'
mkdir -pv /etc/pki/libvirt/private
ln -sv /var/lib/puppet/ssl/certs/ca.pem /etc/pki/CA/cacert.pem
cp -rv /var/lib/puppet/ssl/private_keys/`hostname`.pem /etc/pki/libvirt/private/clientkey.pem
cp -rv /var/lib/puppet/ssl/certs/`hostname`.pem /etc/pki/libvirt/clientcert.pem
chcon -Rv -u system_u /etc/pki/libvirt
chown -Rv root:foreman /etc/pki/libvirt
chmod -Rv ug=rX /etc/pki/libvirt
mkdir -pv /etc/pki/libvirt-vnc
ln -sv /etc/pki/CA/cacert.pem /etc/pki/libvirt-vnc/ca-cert.pem
ln -sv /etc/pki/libvirt/clientcert.pem /etc/pki/libvirt-vnc/client-cert.pem
ln -sv /etc/pki/libvirt/private/client-key.pem /etc/pki/libvirt-vnc/client-key.pem
EOF

echo ' - Updating Compute Resource Libraries'
ssh $ALPHA 'yum -y install foreman-libvirt foreman-gce foreman-compute foreman-vmware foreman-ovirt'

echo ' - Registering Compute Service'
hammer compute_resource create \
	--name='aleph.cluster.ucsf.bkslab.org' \
        --provider='libvirt' \
	--url='qemu+tls://aleph.cluster.ucsf.bkslab.org/system' \
	--description='Aleph Hypervisor (cluster core services)'


echo " - Setting up local resource mirror"
ssh $ALPHA 'yum -y install createrepo'
ssh $ALPHA << 'EOF'
mkdir -pv /var/local/repos


how to set up a hypervisor

  • try using the default method. We use a custom method, but it quickly becomes tcte.
  • theforeman.org

how to set up a new machine

Suggest that we merge many of the sysadmin articles into a small number of related articles.

Retrieved from "http://wiki.docking.org/index.php?title=Hypervisor&oldid=10946"