Hypervisor

ssh into the hypervisor

sudo virsh

list --all

List of hypervisors

YYZ =

• apes
• rocky
• pulp

SFO

This is aleph:/root/setup/libvirt/SETUP

virsh pool-define default-pool.xml
virsh pool-define fast-pool.xml
virsh pool-start fast
virsh pool-autostart fast
virsh pool-define bulk-pool.xml
virsh pool-start bulk
virsh pool-autostart bulk

virsh net-define cluster-network.xml
virsh net-start cluster
virsh net-autostart cluster
virsh net-define public-network.xml
virsh net-start public
virsh net-autostart public

set up first VM

These instructions are from /root/setup/vms/alpha-setup

#!/bin/sh

#TODO:
# Resolv.conf
# Firewalls

CLUSTER_MAC=$( ip addr list br0 | tail -n+2 | head -n1 | tr -s ' ' | cut -d\  -f3 )
CLUSTER_IP=$( ip addr list br0 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f1 )
CLUSTER_PREFIX=$( ip addr list br0 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f2 )
PUBLIC_IP=$( ip addr list br1 | tail -n-1 | tr -s ' ' | cut -d\  -f3 | cut -d/ -f1 )

sed -i "^$PUBLIC_IP " /etc/hosts
echo "$PUBLIC_IP     $( hostname )"  >> /etc/hosts

SHORT_HOSTNAME=$( hostname s )
PUBLIC_HOSTNAME=$( hostname )
PUBLIC_DOMAIN=$( hostname -d )
CLUSTER_DOMAIN=cluster.$PUBLIC_DOMAIN
CLUSTER_HOSTNAME=$SHORT_HOSTNAME.$CLUSTER_DOMAIN
CLUSTER_OCTETS=($( echo $CLUSTER_IP | tr . "\n" ))
CLUSTER_NETWORK=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.0.0
CLUSTER_MASK=$( ifconfig br0 | grep Mask | tr -d ' ' | cut -d: -f4 )
CLUSTER_START=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.0.1
CLUSTER_END=${CLUSTER_OCTETS[0]}.${CLUSTER_OCTETS[1]}.99.99

sed -i "^$CLUSTER_IP " /etc/hosts
sed -i "^$PUBLIC_IP " /etc/hosts
echo "$CLUSTER_IP    $CLUSTER_HOSTNAME" >> /etc/hosts
echo "$PUBLIC_IP     $PUBLIC_HOSTNAME"  >> /etc/hosts

ALPHA=10.20.1.1
EPEL=epel-release-6-8.noarch.rpm

echo " - Generating/Adding ssh-key (Enter password twice)"
ssh-copy-id $ALPHA
eval $( ssh-agent )
ssh-add ~/.ssh/id_rsa

echo " - Transfering repository setup files"
scp ../pkgs/* $ALPHA:~
ssh $ALPHA "yum -y install ~/epel-release-6-8.noarch.rpm"
ssh $ALPHA "mv -v ~/scl.repo /etc/yum.repos.d/scl.repo"
ssh $ALPHA "yum -y update"

echo " - Transfering service setup files"
scp ../services/foreman $ALPHA:~
ssh $ALPHA "sh ~/foreman"

echo " - Configuring Foreman Server"
# Allow: dns, tftp, http, https, nfs, puppet, dhcp
# TODO: network SSH
ssh $ALPHA "lokkit -s dns -s tftp -s http -s https -s nfs -p 8140:tcp -p 67:tcp -p 67:udp "
lokkit -s libvirt -s libvirt-tls

echo " - Hardcoding DNS entries"
ssh $ALPHA "grep alpha /etc/hosts" >> /etc/hostsa


echo " - Retrieving configuration credentials"
scp -r $ALPHA:~/.foreman ~

echo " - Installing local CLI tools"
yum -y install http://yum.theforeman.org/releases/latest/el6/x86_64/foreman-release.rpm
yum -y install rubygem-hammer_cli rubygem-hammer_cli_foreman


echo " - Finalizing Installation (Making Proxy Connections)"
CLUSTER_DOMAIN_ID=$( hammer domain info --name=$CLUSTER_DOMAIN | grep ^Id: | cut -d: -f2 | tr -d ' ' )


hammer host info --id=1
hammer -v host update --id=$CLUSTER_DOAMIN_ID
hammer proxy info --id=1
hammer -v domain update --name=$CLUSTER_DOMAIN --dns-id=1
hammer -v domain update --name=$PUBLIC_DOMAIN --dns-id=1
hammer -v subnet create \
	--name "$CLUSTER_DOMAIN" \
	--network "$CLUSTER_NETWORK" \
	--mask "$CLUSTER_MASK" \
	--dns-primary "$ALPHA" \
	--from "$CLUSTER_START" \
	--to "$CLUSTER_END" \
	--domain-ids "$CLUSTER_DOMAIN_ID" \
	--dhcp-id 1 \
	--dns-id 1 \
	--tftp-id 1

CLUSTER_SUBNET_ID=$( hammer subnet info --name=$CLUSTER_DOMAIN | grep ^Id: | cut -d: -f2 | tr -d ' ' )

echo " - Registering self with Foreman"
# OS is always 1
FOREMAN_OS=$( hammer os info --id 1 | grep ^Id: | cut -d: -f2 | tr -d ' ' )
FOREMAN_ARCH=$( hammer architecture info --name `uname -i` | grep ^Id: | cut -d: -f2 | tr -d ' ' )

hammer -v host create \
	--name "$CLUSTER_HOSTNAME" \
	--ip "$CLUSTER_IP" \
	--mac "$CLUSTER_MAC" \
        --domain-id "$CLUSTER_DOMAIN_ID" \
	--subnet-id "$CLUSTER_SUBNET_ID" \
	--architecture-id "$FOREMAN_ARCH" \
	--operatingsystem-id "$FOREMAN_OS" \
	--managed "true" \
	--build "false"

echo " - Registering with Puppet"
puppet agent --waitforcert 60 --no-daemonize &
ssh $ALPHA "puppet cert sign $( hostname )"
service puppet restart

echo " - Updating Certificates"
mkdir -pv /etc/pki/CA
ln -sv /var/lib/puppet/ssl/certs/ca.pem /etc/pki/CA/cacert.pem
rm -rf /etc/pki/libvirt

mkdir -pv /etc/pki/libvirt/private
cp -rv /var/lib/puppet/ssl/private_keys/`hostname`.pem /etc/pki/libvirt/private/serverkey.pem
cp -rv /var/lib/puppet/ssl/certs/`hostname`.pem /etc/pki/libvirt/servercert.pem 
ln -sv /etc/pki/libvirt/private/serverkey.pem /etc/pki/libvirt/private/clientkey.pem
ln -sv /etc/pki/libvirt/servercert.pem /etc/pki/libvirt/clientcert.pem
chcon -Rv -u system_u /etc/pki/libvirt
chown -Rv root:qemu /etc/pki/libvirt
chmod -Rv 440 /etc/pki/libvirt

mkdir -pv /etc/pki/libvirt-vnc
ln -sv /etc/pki/CA/cacert.pem /etc/pki/libvirt-vnc/ca-cert.pem
ln -sv /etc/pki/libvirt/servercert.pem /etc/pki/libvirt-vnc/server-cert.pem
ln -sv /etc/pki/libvirt/private/serverkey.pem /etc/pki/libvirt-vnc/server-key.pem

sed -i -e '/^listen_tls =/d' '/listen_tcp =/d' /etc/libvirt/libvirtd.conf
echo >> /etc/libvirt/libvirtd.conf << 'EOF'
listen_tls = 1
listen_tcp = 0
EOF

sed -i -e '/vnc_tls =/d' '/vnc_listen =/d' /etc/libvirt/qemu.conf
echo "
vnc_tls = 1
vnc_listen = $CLUSTER_IP
" >> /etc/libvirt/qemu.conf

service libvirtd restart

ssh $ALPHA << 'EOF'
mkdir -pv /etc/pki/libvirt/private
ln -sv /var/lib/puppet/ssl/certs/ca.pem /etc/pki/CA/cacert.pem
cp -rv /var/lib/puppet/ssl/private_keys/`hostname`.pem /etc/pki/libvirt/private/clientkey.pem
cp -rv /var/lib/puppet/ssl/certs/`hostname`.pem /etc/pki/libvirt/clientcert.pem
chcon -Rv -u system_u /etc/pki/libvirt
chown -Rv root:foreman /etc/pki/libvirt
chmod -Rv ug=rX /etc/pki/libvirt
mkdir -pv /etc/pki/libvirt-vnc
ln -sv /etc/pki/CA/cacert.pem /etc/pki/libvirt-vnc/ca-cert.pem
ln -sv /etc/pki/libvirt/clientcert.pem /etc/pki/libvirt-vnc/client-cert.pem
ln -sv /etc/pki/libvirt/private/client-key.pem /etc/pki/libvirt-vnc/client-key.pem
EOF

echo ' - Updating Compute Resource Libraries'
ssh $ALPHA 'yum -y install foreman-libvirt foreman-gce foreman-compute foreman-vmware foreman-ovirt'

echo ' - Registering Compute Service'
hammer compute_resource create \
	--name='aleph.cluster.ucsf.bkslab.org' \
        --provider='libvirt' \
	--url='qemu+tls://aleph.cluster.ucsf.bkslab.org/system' \
	--description='Aleph Hypervisor (cluster core services)'


echo " - Setting up local resource mirror"
ssh $ALPHA 'yum -y install createrepo'
ssh $ALPHA << 'EOF'
mkdir -pv /var/local/repos

how to set up a hypervisor

• try using the default method. We use a custom method, but it quickly becomes tcte.
• theforeman.org

how to set up a new machine

Suggest that we merge many of the sysadmin articles into a small number of related articles.