From 8e2815bddcd7a1e621d9ce1e38bd72ae05a12783 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Franti=C5=A1ek=20Dvo=C5=99=C3=A1k?= Date: Sun, 17 Jan 2016 21:42:44 +0100 Subject: [PATCH] Docker tuned (IPv4 private with VXLAN, IPv6 public). --- HOWTO-docker.sh | 126 +++++++++++++++++++++++++++++++++++++++++-- docker.sh | 126 +++++++++++++++++++++++++++++++++++-------- docker/Dockerfile.meta | 16 +++++- docker/Dockerfile.puppetlabs | 7 +++ hador-gen-virt.sh | 29 ++++++++++ mac2ip6.sh | 42 +++++++++++++++ 6 files changed, 318 insertions(+), 28 deletions(-) create mode 100755 hador-gen-virt.sh create mode 100755 mac2ip6.sh diff --git a/HOWTO-docker.sh b/HOWTO-docker.sh index 1bee4f8..7e786c2 100644 --- a/HOWTO-docker.sh +++ b/HOWTO-docker.sh @@ -27,12 +27,132 @@ Installation ============ #! /bin/sh +apt-get install apt-transport-https + cat > /etc/apt/sources.list.d/docker.list < /etc/apt/sources.list.d/backports.list apt-get update -apt-get install docker-engine -echo "DOCKER_OPTS='--ipv6 --fixed-cidr-v6=2001:718:ff01:1::/64'" >> /etc/default/docker +apt-get install docker-engine bridge-utils +#echo "DOCKER_OPTS='...'" >> /etc/default/docker + + +Network experiments +=================== + +# check ipv6: +sysctl net.ipv6.conf.all.disable_ipv6 net.ipv6.conf.default.disable_ipv6 net.ipv6.conf.lo.disable_ipv6 net.ipv6.conf.all.forwarding net.ipv6.conf.default.forwarding +# enable ipv6 (temporary): +sysctl net.ipv6.conf.all.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.lo.disable_ipv6=0 +sysctl net.ipv6.conf.all.forwarding=1 net.ipv6.conf.default.forwarding=1 + +brctl addbr docker1 +#ip addr add 192.168.5.1/24 dev docker1 +ip -6 addr add 2001:718:ff01:1:a:b:c:d/64 dev docker1 # ip opsat z link adresy +ip link set dev docker1 up + +#not with autoconfig: route -6 add default gw 2001:718:ff01:1::1 dev br0 + +# IPv6-only is not possible in docker, better to do it manually +#docker network create -d bridge --subnet=2001:718:ff01:1::/64 --gateway 2001:718:ff01:1::1 -o com.docker.network.bridge.name=docker1 -o com.docker.network.bridge.enable_ip_masquerade=false public6 +#docker network inspect public6 +#docker network connect public6 $CONTAINER_ID +#docker network disconnect public6 $CONTAINER_ID + +# po vyrobení stroje: +IP6=docker inspect $CONTAINER_ID +route -6 add $IP6 dev docker0 + + +wget https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework -P /usr/local/bin/ +chmod +x /usr/local/bin/pipework +pipework docker1 $CONTAINER_ID 2001:718:ff01:1::1/64@$IPV6 + +============= + +dvojice adres: jedna pro bridge (mohla by být čtveřice..) + +#/etc/default/docker: +#DOCKER_OPTS="--bridge=br0 --fixed-cidr='10.4.3.4/31' --ip-masq=false --iptables=false --ipv6=false" +DOCKER_OPTS="--bridge=br0 --fixed-cidr='10.4.3.4/31' --ipv6=false" + +#/etc/network/interfaces: +auto br0 + +allow-hotplug eth1 +iface eth1 inet manual + +allow-hotplug eth2 +iface eth2 inet manual + +iface br0 inet static + address 10.4.3.4 + netmask 255.255.255.0 + #gateway 10.4.3.1 + bridge_fd 0 + bridge_maxwait 0 + bridge_stp off + bridge_ports eth1 + +# funguje samo: +#iface br0 inet6 static +# address 2001:718:ff02::aff:fe04:304 +# netmask 64 + +# setup: +pid=`docker inspect -f '{{.State.Pid}}' hadoop1` +i6prefix='2001:718:ff02:1' +mkdir -p /var/run/netns || : +ln -s /proc/$pid/ns/net /var/run/netns/$pid || : +ip netns exec $pid ip -6 addr add ${ip6prefix}:42:aff:fe04:305/64 dev eth0 +ip netns exec $pid ip -6 route add default via ${ip6prefix}::1 +rm -rf /var/run/netns +#find -L /var/run/netns -type l -delete + +================= + +# veřejné adresy na totožném rozhraní +# * docker by normálně rozbil, ale s timhle nastavením nerozbil (=bez NATu) +# * gateway pro containery je třeba uvézt, jinak by to byla adresa br0 +/etc/default/docker: +DOCKER_OPTS="--bridge=br0 --fixed-cidr=147.251.9.220/31 --default-gateway=147.251.9.1 --ip-masq=false --iptables=false --ipv6=false" + +Network overlay +=============== + +# vyžaduje jadro >= 3.16 + +/etc/default/docker: +DOCKER_OPTS="--cluster-advertise=br0:2376 --cluster-store=zk://hador-c1.ics.muni.cz,hador-c2.ics.muni.cz,hador.ics.muni.cz/docker-hador" +DOCKER_OPTS="--cluster-advertise=eth0:2376 --cluster-store=zk://hador-c1.ics.muni.cz,hador-c2.ics.muni.cz,hador.ics.muni.cz/docker-cloud" + +docker network create -d overlay mn +docker run ... --net=mn $IMAGE $CMD +docker network connect bridge $CONTAINER_ID + +#pak IPv6 ručně +pid=`docker inspect -f '{{.State.Pid}}' hadoop1` +ip6prefix='2001:718:ff02:1' +mkdir -p /var/run/netns || : +ln -s /proc/$pid/ns/net /var/run/netns/$pid || : +ip link add veth1a type veth peer name veth1b +brctl addif br0 veth1a +ip link set veth1a up +ip link set veth1b netns $pid +ip netns exec $pid ip link set dev veth1b name eth6 +#auto?:ip netns exec $pid ip link set eth6 address 12:34:56:78:9a:bc +ip netns exec $pid ip link set eth6 up +#auto?:ip netns exec $pid ip -6 addr add ${ip6prefix}:42:aff:fe04:305/64 dev eth6 +#auto:ip netns exec $pid ip -6 route add default via ${ip6prefix}::1 +rm -rf /var/run/netns + + +==== Troubleshooting ==== + +#docker: Error response from daemon: Could not find container for entity id ... +/var/lib/docker/linkgraph.db +service docker restart diff --git a/docker.sh b/docker.sh index 7c9c23a..8defd42 100755 --- a/docker.sh +++ b/docker.sh @@ -1,47 +1,127 @@ -#! /bin/sh -xe +#! /bin/sh -e + +# +# Script to create docker machine: +# * IPv4 private overlay network (docker overlay) +# * IPv4 private network with NAT (docker bridge) +# * IPv6 public network directly (manually, with autoconfiguration) +# +# Reqirements: +# +# 0) docker from upstream + backports (docker-engine 1.10, linux kernel >= 3.16) +# +# apt-get install linux-image-amd64 linux-headers-amd64 -t wheezy-backports +# apt-get install docker-engine bridge-utils -t wheezy-backports +# +# 1) zookeeper (public, or port mapped with NAT) +# +# 2) for IPv6: IPv6 autoconfiguration enabled +# +# 3) /etc/default/docker +# +# DOCKER_OPTS="--cluster-advertise=br0:2376 --cluster-store=zk://hador-c1.ics.muni.cz,hador-c2.ics.muni.cz,hador.ics.muni.cz/docker-hador" +# +# 4) docker overlay network created +# +# docker network create -d overlay vxlan +# + +#DOCKER_network='vxlan' +DOCKER_network='bridge' + + +# get the first free network device +devname() { + i=1 + while ip addr show dev veth${i}a >/dev/null 2>&1; do + i=`expr $i + 1` + done + + echo ${i} +} -#./launch.sh cfg.sh --name=hadoop1 --hostname=`hostname`-hadoop1 --mac-address=02:42:ac:11:00:02 --memory=61440G if [ -z "${1}" ]; then echo "Usage:" echo - echo "$0 cfg_hostname.sh" + echo "$0 cfg_hostname.sh [init] []" exit 1 fi . ./$1 || exit 1 shift if [ -n "${FACTER_hostname}" ]; then - ARGS="${ARGS} --name=${FACTER_hostname} --hostname=${FACTER_hostname}" + ARGS="${ARGS} --name=${FACTER_hostname} --hostname=${FACTER_fqdn}" fi if [ -n "${FACTER_macaddress}" ]; then - ARGS="${ARGS} --mac-address=${FACTER_macaddress}M" + ARGS="${ARGS} --mac-address=${FACTER_macaddress}" fi -if [ -n "${FACTER_macaddress}" ]; then +if [ -n "${SIZE_MEM}" ]; then ARGS="${ARGS} --memory=${SIZE_MEM}M" fi if [ -z "${DISK_SUBDIR}" ]; then DISK_SUBDIR='/virt-1' fi -echo docker run -itd \ +# disks +if [ x"${1}" = x"init" ]; then + shift + + docker pull valtri/hadoop:puppetlabs + i=0 + for d in ${DISKS}; do + i=$((i+1)) + mkdir /data/${i}${DISK_SUBDIR} 2>/dev/null || : + done + mkdir /scratch${DISK_SUBDIR} 2>/dev/null || : + + if [ ! -f ~/.ssh/id_rsa_docker ]; then + ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa_docker + cp -p ~/.ssh/id_rsa_docker.pub ~/.ssh/authorized_keys_docker + if [ -f `dirname $0`/authorized_keys ] + cat `dirname $0`/authorized_keys >> ~/.ssh/authorized_keys_docker + fi + fi +fi +i=0 +for d in ${DISKS}; do + i=$((i+1)) + ARGS="${ARGS} -v /data/${i}${DISK_SUBDIR}:/data/${i}" +done +ARGS="${ARGS} -v /scratch:/scratch" + +docker run -itd \ -v ~/.k5login:/root/.k5login \ - -v ~/.ssh:/root/.ssh \ + -v ~/.ssh/authorized_keys_docker:/root/.ssh/authorized_keys \ -v /etc/krb5.conf:/etc/krb5.conf \ - -v /data/1${DISK_SUBDIR}:/data/1 \ - -v /data/2${DISK_SUBDIR}:/data/2 \ - -v /data/3${DISK_SUBDIR}:/data/3 \ - -v /data/4${DISK_SUBDIR}:/data/4 \ - -v /data/5${DISK_SUBDIR}:/data/5 \ - -v /data/6${DISK_SUBDIR}:/data/6 \ - -v /data/7${DISK_SUBDIR}:/data/7 \ - -v /data/8${DISK_SUBDIR}:/data/8 \ - -v /data/9${DISK_SUBDIR}:/data/9 \ - -v /data/10${DISK_SUBDIR}:/data/10 \ - -v /data/11${DISK_SUBDIR}:/data/11 \ - -v /data/12${DISK_SUBDIR}:/data/12 \ - -v /scratch${DISK_SUBDIR}:/scratch \ - "${ARGS}" \ + --net=${DOCKER_network} \ + --restart=on-failure:5 \ + ${ARGS} \ "$@" \ - hadoop-meta \ + valtri/hadoop:puppetlabs \ /sbin/init + +# ==== public IPv4 ==== +if [ x"${DOCKER_network}" != x"bridge" ]; then + docker network connect bridge ${FACTER_hostname} +fi + +# ==== public IPv6 ==== +dev=veth`devname` +# this is not persistent, let's create a script +cat << EOF > /var/run/docker/${FACTER_hostname}.sh +mkdir -p /var/run/netns || : +find -L /etc/ssl/certs -type l -delete || : +pid=\`docker inspect -f '{{.State.Pid}}' ${FACTER_hostname}\` + +ln -s /proc/\$pid/ns/net /var/run/netns/\$pid || : +ip link add ${dev}a type veth peer name ${dev}b +brctl addif ${XENBR} ${dev}a +ip link set ${dev}a up +ip link set ${dev}b netns \$pid +ip netns exec \$pid ip link set dev ${dev}b name eth6 +ip netns exec \$pid ip link set eth6 address ${FACTER_macaddress} +ip netns exec \$pid ip link set eth6 up +EOF +chmod +x /var/run/docker/${FACTER_hostname}.sh +sh -xe /var/run/docker/${FACTER_hostname}.sh diff --git a/docker/Dockerfile.meta b/docker/Dockerfile.meta index 5538222..8a30f60 100644 --- a/docker/Dockerfile.meta +++ b/docker/Dockerfile.meta @@ -1,10 +1,22 @@ FROM debian:7 MAINTAINER František Dvořák +# ==== system ==== + +RUN echo 'locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8' | debconf-set-selections +RUN echo 'locales locales/default_environment_locale select en_US.UTF-8' | debconf-set-selections RUN apt-get update \ -&& apt-get install -y openssh-server puppet +&& apt-get install locales + +# ==== ssh ==== + +RUN apt-get update \ +&& apt-get install -y openssh-server \ +&& sed -e 's/^#\(GSSAPIAuthentication\).*/\1 yes/' -i /etc/ssh/sshd_config + +# ==== puppet ==== -RUN sed -e 's/^#\(GSSAPIAuthentication\).*/\1 yes/' -i /etc/ssh/sshd_config +RUN apt-get update && apt-get install puppet RUN sed -e 's/\(\[main\]\)/\1\nserver=thorin2.ics.muni.cz/' -i /etc/puppet/puppet.conf # ==== cleanup ==== diff --git a/docker/Dockerfile.puppetlabs b/docker/Dockerfile.puppetlabs index 70ef149..11354ab 100644 --- a/docker/Dockerfile.puppetlabs +++ b/docker/Dockerfile.puppetlabs @@ -1,6 +1,13 @@ FROM debian:7 MAINTAINER František Dvořák +# ==== system ==== + +RUN echo 'locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8' | debconf-set-selections +RUN echo 'locales locales/default_environment_locale select en_US.UTF-8' | debconf-set-selections +RUN apt-get update \ +&& apt-get install locales + # ==== ssh ==== RUN apt-get update \ diff --git a/hador-gen-virt.sh b/hador-gen-virt.sh new file mode 100755 index 0000000..fd97e79 --- /dev/null +++ b/hador-gen-virt.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +N=$1 +NVIRT=$2 + +if [ -z "${N}" ]; then + echo "Usage: $0 []" + exit 1 +fi +if [ -z "${NVIRT}" ]; then + NVIRT=1 +fi + +N_HEX=`printf "%02x\n" ${N}` +NVIRT_HEX=`printf "%02x\n" ${NVIRT}` + +cat <