Hypervisor: Difference between revisions

From DISI
Jump to navigation Jump to search
(→‎List of hypervisors: Added vav to the chart)
Line 106: Line 106:
|-
|-
| lamed  || 2 || 0 || 5 ||  none || SiM 4-in-1 32-way
| lamed  || 2 || 0 || 5 ||  none || SiM 4-in-1 32-way
|-
| vav || 2 || 0 || 6 || ntpd || machine DL160G5
|}
|}



Revision as of 00:02, 7 June 2017

ssh to hypervisor, run commands

sudo virsh
list --all
start (full name)
shutdown (number)

Configuration Steps

configure/test public network

edit ifcfg-eth1 - public network

create network bridges

ifcfg-br0 (private network)

DEVICE="br0"
TYPE="Bridge"
ONBOOT="yes"
BOOTPROTO="dhcp"
DHCP_HOSTNAME="tet.cluster.ucsf.bkslab.org"
HOSTNAME="tet.cluster.ucsf.bkslab.org"
DOMAIN="cluster.ucsf.bkslab.org"
STP="off"
DELAY="0"

ifcfg-br1 - public network

DEVICE="br1"
TYPE="Bridge"
ONBOOT="yes"
NM_CONTROLLED="no"
BOOTPROTO="static"
IPADDR="169.230.26.189"
PREFIX="24"
GATEWAY="169.230.26.1"
STP="off"
DELAY="0"

libvirt to define networks

yum install libvirt
chkconfig --levels=345 libvirtd on
service libvirtd start

mkdir ~/libvirt
<get default files>
virsh net-define cluster-network.xml
virsh net-start cluster
virsh net-autostart cluster

virsh net-define public-network.xml
virsh net-start public
virsh net-autostart public

virsh pool-define default-pool.xml
virsh pool-start default
virsh pool-autostart default

echo "
vnc_tls = 1
vnc_tls_x509_verify = 1
vnc_tls_x509_cert_dir = "/etc/pki/libvirt-vnc"
vnc_listen = "<CLUSTER IP ADDRESS>"
vnc_password = "4nu6rho3"
user = "qemu"
group = "qemu"
" >> /etc/libvirtd/qemu.conf

echo "
log_level = 1
auth_tcp = "none"
log_outputs="3:syslog:libvirtd"
listen_tls = 1
listen_tcp = 0
" >> /etc/libvirtd/libvirtd.conf

echo 'LIBVIRTD_ARGS="--listen"' >> /etc/sysconfig/libvirtd


#below command wipes out iptables.  be careful.
lokkit -s libvirtd-tls

List of hypervisors

Hypervisor Name Cluster Rack # Hypervisor # Runs Machine
vmware1 0 1 1 mostly linux services DL165G5 ??
vmware2 0 1 2 windows, obsolete DL165G5 ??
apes 1 1 1 ?? 64 way C6145
rocky 1 1 6 lots... 64 way C6145
pulp 1 1 2 lots... 64 Way C6145
aleph 2 0 1 alpha,beta,gamma 8 way DL160g5
tet 2 0 3 Matt's machine SiM 4-in-1 32-way
yod 2 0 2 none SiM 4-in-1 32-way
kaf 2 0 4 none SiM 4-in-1 32-way
lamed 2 0 5 none SiM 4-in-1 32-way
vav 2 0 6 ntpd machine DL160G5

Set up hypervisor

This is aleph:/root/setup/libvirt/SETUP

virsh pool-define default-pool.xml
virsh pool-define fast-pool.xml
virsh pool-start fast
virsh pool-autostart fast
virsh pool-define bulk-pool.xml
virsh pool-start bulk
virsh pool-autostart bulk

virsh net-define cluster-network.xml
virsh net-start cluster
virsh net-autostart cluster
virsh net-define public-network.xml
virsh net-start public
virsh net-autostart public

BKS SGE Cell

https://alpha.compbio.ucsf.edu/config_templates/43-todo-sge-execd/edit

set up first VM

These instructions are from /root/setup/vms/alpha-setup

#!/bin/sh

#TODO:
# Resolv.conf
# Firewalls

CLUSTER_MAC=$( ip addr list br0 | tail -n+2 | head -n1 | tr -s ' ' | cut -d\  -f3 )
CLUSTER_IP=$( ip addr list br0 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f1 )
CLUSTER_PREFIX=$( ip addr list br0 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f2 )
PUBLIC_IP=$( ip addr list br1 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f1 )

sed -i "^$PUBLIC_IP " /etc/hosts
echo "$PUBLIC_IP     $( hostname )"  >> /etc/hosts

SHORT_HOSTNAME=$( hostname s )
PUBLIC_HOSTNAME=$( hostname )
PUBLIC_DOMAIN=$( hostname -d )
CLUSTER_DOMAIN=cluster.$PUBLIC_DOMAIN
CLUSTER_HOSTNAME=$SHORT_HOSTNAME.$CLUSTER_DOMAIN
CLUSTER_OCTETS=($( echo $CLUSTER_IP | tr . "\n" ))
CLUSTER_NETWORK=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.0.0
CLUSTER_MASK=$( ifconfig br0 | grep Mask | tr -d ' ' | cut -d: -f4 )
CLUSTER_START=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.0.1
CLUSTER_END=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.99.99

sed -i "^$CLUSTER_IP " /etc/hosts
sed -i "^$PUBLIC_IP " /etc/hosts
echo "$CLUSTER_IP    $CLUSTER_HOSTNAME" >> /etc/hosts
echo "$PUBLIC_IP     $PUBLIC_HOSTNAME"  >> /etc/hosts

ALPHA=10.20.1.1
EPEL=epel-release-6-8.noarch.rpm

echo " - Generating/Adding ssh-key (Enter password twice)"
ssh-copy-id $ALPHA
eval $( ssh-agent )
ssh-add ~/.ssh/id_rsa

echo " - Transfering repository setup files"
scp ../pkgs/* $ALPHA:~
ssh $ALPHA "yum -y install ~/epel-release-6-8.noarch.rpm"
ssh $ALPHA "mv -v ~/scl.repo /etc/yum.repos.d/scl.repo"
ssh $ALPHA "yum -y update"

echo " - Transfering service setup files"
scp ../services/foreman $ALPHA:~
ssh $ALPHA "sh ~/foreman"

echo " - Configuring Foreman Server"
# Allow: dns, tftp, http, https, nfs, puppet, dhcp
# TODO: network SSH
ssh $ALPHA "lokkit -s dns -s tftp -s http -s https -s nfs -p 8140:tcp -p 67:tcp -p 67:udp "
lokkit -s libvirt -s libvirt-tls

echo " - Hardcoding DNS entries"
ssh $ALPHA "grep alpha /etc/hosts" >> /etc/hostsa


echo " - Retrieving configuration credentials"
scp -r $ALPHA:~/.foreman ~

echo " - Installing local CLI tools"
yum -y install http://yum.theforeman.org/releases/latest/el6/x86_64/foreman-release.rpm
yum -y install rubygem-hammer_cli rubygem-hammer_cli_foreman


echo " - Finalizing Installation (Making Proxy Connections)"
CLUSTER_DOMAIN_ID=$( hammer domain info --name=$CLUSTER_DOMAIN | grep ^Id: | cut -d: -f2 | tr -d ' ' )


hammer host info --id=1
hammer -v host update --id=$CLUSTER_DOAMIN_ID
hammer proxy info --id=1
hammer -v domain update --name=$CLUSTER_DOMAIN --dns-id=1
hammer -v domain update --name=$PUBLIC_DOMAIN --dns-id=1
hammer -v subnet create \
	--name "$CLUSTER_DOMAIN" \
	--network "$CLUSTER_NETWORK" \
	--mask "$CLUSTER_MASK" \
	--dns-primary "$ALPHA" \
	--from "$CLUSTER_START" \
	--to "$CLUSTER_END" \
	--domain-ids "$CLUSTER_DOMAIN_ID" \
	--dhcp-id 1 \
	--dns-id 1 \
	--tftp-id 1

CLUSTER_SUBNET_ID=$( hammer subnet info --name=$CLUSTER_DOMAIN | grep ^Id: | cut -d: -f2 | tr -d ' ' )

echo " - Registering self with Foreman"
# OS is always 1
FOREMAN_OS=$( hammer os info --id 1 | grep ^Id: | cut -d: -f2 | tr -d ' ' )
FOREMAN_ARCH=$( hammer architecture info --name `uname -i` | grep ^Id: | cut -d: -f2 | tr -d ' ' )

hammer -v host create \
	--name "$CLUSTER_HOSTNAME" \
	--ip "$CLUSTER_IP" \
	--mac "$CLUSTER_MAC" \
        --domain-id "$CLUSTER_DOMAIN_ID" \
	--subnet-id "$CLUSTER_SUBNET_ID" \
	--architecture-id "$FOREMAN_ARCH" \
	--operatingsystem-id "$FOREMAN_OS" \
	--managed "true" \
	--build "false"

echo " - Registering with Puppet"
puppet agent --waitforcert 60 --no-daemonize &
ssh $ALPHA "puppet cert sign $( hostname )"
service puppet restart

echo " - Updating Certificates"
mkdir -pv /etc/pki/CA
ln -sv /var/lib/puppet/ssl/certs/ca.pem /etc/pki/CA/cacert.pem
rm -rf /etc/pki/libvirt

mkdir -pv /etc/pki/libvirt/private
cp -rv /var/lib/puppet/ssl/private_keys/`hostname`.pem /etc/pki/libvirt/private/serverkey.pem
cp -rv /var/lib/puppet/ssl/certs/`hostname`.pem /etc/pki/libvirt/servercert.pem 
ln -sv /etc/pki/libvirt/private/serverkey.pem /etc/pki/libvirt/private/clientkey.pem
ln -sv /etc/pki/libvirt/servercert.pem /etc/pki/libvirt/clientcert.pem
chcon -Rv -u system_u /etc/pki/libvirt
chown -Rv root:qemu /etc/pki/libvirt
chmod -Rv 440 /etc/pki/libvirt

mkdir -pv /etc/pki/libvirt-vnc
ln -sv /etc/pki/CA/cacert.pem /etc/pki/libvirt-vnc/ca-cert.pem
ln -sv /etc/pki/libvirt/servercert.pem /etc/pki/libvirt-vnc/server-cert.pem
ln -sv /etc/pki/libvirt/private/serverkey.pem /etc/pki/libvirt-vnc/server-key.pem

sed -i -e '/^listen_tls =/d' '/listen_tcp =/d' /etc/libvirt/libvirtd.conf
echo >> /etc/libvirt/libvirtd.conf << 'EOF'
listen_tls = 1
listen_tcp = 0
EOF

sed -i -e '/vnc_tls =/d' '/vnc_listen =/d' /etc/libvirt/qemu.conf
echo "
vnc_tls = 1
vnc_listen = $CLUSTER_IP
" >> /etc/libvirt/qemu.conf

service libvirtd restart

ssh $ALPHA << 'EOF'
mkdir -pv /etc/pki/libvirt/private
ln -sv /var/lib/puppet/ssl/certs/ca.pem /etc/pki/CA/cacert.pem
cp -rv /var/lib/puppet/ssl/private_keys/`hostname`.pem /etc/pki/libvirt/private/clientkey.pem
cp -rv /var/lib/puppet/ssl/certs/`hostname`.pem /etc/pki/libvirt/clientcert.pem
chcon -Rv -u system_u /etc/pki/libvirt
chown -Rv root:foreman /etc/pki/libvirt
chmod -Rv ug=rX /etc/pki/libvirt
mkdir -pv /etc/pki/libvirt-vnc
ln -sv /etc/pki/CA/cacert.pem /etc/pki/libvirt-vnc/ca-cert.pem
ln -sv /etc/pki/libvirt/clientcert.pem /etc/pki/libvirt-vnc/client-cert.pem
ln -sv /etc/pki/libvirt/private/client-key.pem /etc/pki/libvirt-vnc/client-key.pem
EOF

echo ' - Updating Compute Resource Libraries'
ssh $ALPHA 'yum -y install foreman-libvirt foreman-gce foreman-compute foreman-vmware foreman-ovirt'

echo ' - Registering Compute Service'
hammer compute_resource create \
	--name='aleph.cluster.ucsf.bkslab.org' \
        --provider='libvirt' \
	--url='qemu+tls://aleph.cluster.ucsf.bkslab.org/system' \
	--description='Aleph Hypervisor (cluster core services)'


echo " - Setting up local resource mirror"
ssh $ALPHA 'yum -y install createrepo'
ssh $ALPHA << 'EOF'
mkdir -pv /var/local/repos


how to set up a hypervisor

  • try using the default method. We use a custom method, but it quickly becomes tcte.
  • theforeman.org

how to set up a new machine

Suggest that we merge many of the sysadmin articles into a small number of related articles.