diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e6554e --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +hack/*.txt + +hack/kubernetes-clearwater/*.gpg +hack/kubernetes-clearwater/openshift* + +hack/**/http%3A%2F%2Frepo.cw-ngv.com%2Frepo-key +hack/**/*.tangfx + +hack/kubernetes-clearwater/clearwater/*.txt diff --git a/base/liveness.sh b/base/liveness.sh new file mode 100644 index 0000000..a0d1e4a --- /dev/null +++ b/base/liveness.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# Liveness checker; tests whether this instance is live +set -e +PORTS=${1:-none} +ADDRESS=${MY_POD_IP:-127.0.0.1} +TIMEOUT=${2:-5} + +if [ "$PORTS" = "none" ] +then + echo "PORTS is not defined" + exit 1 +fi + +echo "Reporting config" +echo " ADDRESS=${ADDRESS}" +echo " PORTS=${PORTS}" +echo " TIMEOUT=${TIMEOUT}" + +for PORT in $PORTS +do + echo "Testing port ${PORT}" + nc -n -z -w ${TIMEOUT} ${ADDRESS} ${PORT} + echo " port ${PORT} OK" +done + +echo "Got to end - all must be OK" +exit 0 diff --git a/hack.md b/hack.md new file mode 100644 index 0000000..329f453 --- /dev/null +++ b/hack.md @@ -0,0 +1,27 @@ +### About hack + +[Deploy _clearwater_ into _kubernetes_](./hack/kubernetes-clearwater) + +## Preferences of SIP windows apps + +To overview more apps, refer to http://clearwater.readthedocs.io/en/latest/Making_your_first_call.html#prerequisites + +To overview more app pictures, step into [./hack/sip-apps](./hack/sip-apps) + +### Blink + +* Windows 10 + +![win10](./hack/sip-apps/blink.png) + +### X-Lite + +* Windows 10 + +![win10](./hack/sip-apps/x-lite.png) + +### Zoiper + +* Windows 10 + +![win10](./hack/sip-apps/zoiper.png) \ No newline at end of file diff --git a/hack/README.md b/hack/README.md new file mode 100755 index 0000000..110c263 --- /dev/null +++ b/hack/README.md @@ -0,0 +1,221 @@ +About +====== + +Tables of Content +----------------- + +* Background + +* POC + +* Staging + +Background +----------- + +The R&D target is to depolying Metaswitch ClearWater (IMS) into Kubernetes（v1.3+） + +Cluster： + +* 3 nodes CentOS 7 in Private Data center + +* Physical networking： + +Project： + +* Inspired by Metaswitch clearwater-docker repository + +从github克隆fork仓库 + + tangf@DESKTOP-H68OQDV /cygdrive/g/work/src/github.com + $ git clone https://github.com/stackdocker/clearwater-docker Metaswitch/clearwater-docker + 正克隆到 'Metaswitch/clearwater-docker'... + remote: Counting objects: 431, done. + remote: Total 431 (delta 0), reused 0 (delta 0), pack-reused 431 + 接收对象中: 100% (431/431), 114.77 KiB | 57.00 KiB/s, 完成. + 处理 delta 中: 100% (204/204), 完成. + 检查连接... 完成。 + + tangf@DESKTOP-H68OQDV /cygdrive/g/work/src/github.com + $ cd Metaswitch/clearwater-docker/ + + tangf@DESKTOP-H68OQDV /cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker + $ git remote -v + origin https://github.com/stackdocker/clearwater-docker (fetch) + origin https://github.com/stackdocker/clearwater-docker (push) + +Join community，for pull request available + + tangf@DESKTOP-H68OQDV /cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker + $ git remote add upstream https://github.com/Metaswitch/clearwater-docker + + tangf@DESKTOP-H68OQDV /cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker + $ git remote -v + origin https://github.com/stackdocker/clearwater-docker (fetch) + origin https://github.com/stackdocker/clearwater-docker (push) + upstream https://github.com/Metaswitch/clearwater-docker (fetch) + upstream https://github.com/Metaswitch/clearwater-docker (push) + +POC +----- +POC是开发者在Staging环境内无法或受限下（例如internet），而在Staging环境外（如个人开发计算机，其它云端）按Staging相同架构而组建的开发环境：cluster的操作系统是一致的，cluster的网络结构是一致的。 + +### 使用virtualbox和vagrant平台 + +Reference: + +* [virtualbox 5.0.28](https://www.virtualbox.org/wiki/Download_Old_Builds_5_0) + +* [vagrant 1.8.6](https://releases.hashicorp.com/vagrant/1.8.6/) + +* [centos7.2.1609 box](https://atlas.hashicorp.com/centos/boxes/7/versions/1609.01) + +注：以上开发工具的版本，是基于当前的实践，如果在后续版本上也实践证明了的，可merge到该README中 + +Getting started： + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ vagrant init --box-version=v1609.01 centos/7 + A `Vagrantfile` has been placed in this directory. You are now + ready to `vagrant up` your first virtual environment! Please read + the comments in the Vagrantfile as well as documentation on + `vagrantup.com` for more information on using Vagrant. + +编辑Vagrantfile文件，包括CentOS box版本，hostonly-network，shared-fs，memory + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ vim Vagrantfile + + config.vm.box = "centos/7" + config.vm.box_version = "1609.01" + + # config.vm.network "private_network", ip: "192.168.33.10" + config.vm.network "private_network", ip: "10.64.33.81", auto_config: false + + # config.vm.synced_folder "../data", "/vagrant_data" + config.vm.synced_folder "G:/work", "/work", disabled: true + + # View the documentation for the provider you are using for more + # information on available options. + config.vm.provider "virtualbox" do |vb| + vb.memory = 4096 + vb.auto_nat_dns_proxy = false + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + end + + config.ssh.insert_key = false + +Provision VM + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ vagrant up --provider virtualbox + Bringing machine 'default' up with 'virtualbox' provider... + ==> default: Box 'centos/7' could not be found. Attempting to find and install... + default: Box Provider: virtualbox + default: Box Version: 1609.01 + ==> default: Loading metadata for box 'centos/7' + default: URL: https://atlas.hashicorp.com/centos/7 + ==> default: Adding box 'centos/7' (v1609.01) for provider: virtualbox + default: Downloading: https://atlas.hashicorp.com/centos/boxes/7/versions/1609.01/providers/virtualbox.box + ==> default: Box download is resuming from prior download progress + default: Progress: 22% (Rate: 918k/s, Estimated time remaining: 0:04:57) + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ vagrant ssh + Last login: Mon Nov 7 11:36:53 2016 from 10.0.2.2 + [vagrant@localhost ~]$ ip a + 1: lo: mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: eth0: mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether 52:54:00:9f:bd:fd brd ff:ff:ff:ff:ff:ff + inet 10.0.2.15/24 brd 10.0.2.255 scope global dynamic eth0 + valid_lft 86081sec preferred_lft 86081sec + inet6 fe80::5054:ff:fe9f:bdfd/64 scope link + valid_lft forever preferred_lft forever + 3: eth1: mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether 08:00:27:c7:35:fe brd ff:ff:ff:ff:ff:ff + inet 10.64.33.81/24 brd 10.64.33.255 scope global eth1 + valid_lft forever preferred_lft forever + inet6 fe80::a00:27ff:fec7:35fe/64 scope link + valid_lft forever preferred_lft forever + + [vagrant@localhost ~]$ cat /etc/redhat-release + CentOS Linux release 7.2.1511 (Core) + + [vagrant@localhost ~]$ uname -r + 3.10.0-327.36.1.el7.x86_64 + + [vagrant@localhost ~]$ sudo df -h + Filesystem Size Used Avail Use% Mounted on + /dev/mapper/VolGroup00-LogVol00 38G 884M 37G 3% / + devtmpfs 1.4G 0 1.4G 0% /dev + tmpfs 1.4G 0 1.4G 0% /dev/shm + tmpfs 1.4G 8.3M 1.4G 1% /run + tmpfs 1.4G 0 1.4G 0% /sys/fs/cgroup + /dev/sda2 497M 126M 372M 26% /boot + tmpfs 285M 0 285M 0% /run/user/1000 + + [vagrant@localhost ~]$ ls /dev/sd* + /dev/sda /dev/sda1 /dev/sda2 /dev/sda3 + + [vagrant@localhost ~]$ sudo pvdisplay + --- Physical volume --- + PV Name /dev/sda3 + VG Name VolGroup00 + PV Size 39.51 GiB / not usable 10.00 MiB + Allocatable yes + PE Size 32.00 MiB + Total PE 1264 + Free PE 10 + Allocated PE 1254 + PV UUID oUVt67-9kA8-uxSc-e6rl-J6ER-unza-Yuc0IA + +Issue + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ /cygdrive/g/app/Oracle/VirtualBox/VBoxManage.exe list runningvms + "origin_openshiftdev_1470677374340_33624" {c3c6bb78-ecec-4c13-922a-c146eb76fc35} + "7_default_1478518426514_80470" {c1bde9cb-97f4-4f91-94d2-6b125a13dd44} + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ vagrant halt + ==> default: Attempting graceful shutdown of VM... + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ /cygdrive/g/app/Oracle/VirtualBox/VBoxManage.exe modifyvm "7_default_1478518426514_80470" --natdnshostresolver1 on + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ /cygdrive/g/app/Oracle/VirtualBox/VBoxManage.exe modifyvm "7_default_1478518426514_80470" --natdnsproxy1 off + +### Details + +Mainly in [kubernetes-clearwater](./kubernetes-clearwater) path. + +* CentOS DevOps + +* Docker v1.10.3 installation + +* Etcd v2 installation + +* Flannel installation + +* Kubernetes v1.3 installation + +* Docker registry 2.4 installation + +Some scratch operation docs are in [kubernetes-clearwater/Kubernetes](./kubernetes-clearwater/Kubernetes) + +Staging +--------- + +Documentations are archived in [k8s-cw-staging-cmri](./k8s-cw-staging-cmri) path. + +* CentOS DevOps + +* K8s v1.3 DevOps + +* ClearWater k8s DevOps diff --git a/hack/call-together-blink-zoiper-xlite.png b/hack/call-together-blink-zoiper-xlite.png new file mode 100755 index 0000000..5233312 Binary files /dev/null and b/hack/call-together-blink-zoiper-xlite.png differ diff --git a/hack/k8s-cw-staging-cmri/README.md b/hack/k8s-cw-staging-cmri/README.md new file mode 100644 index 0000000..68b9635 --- /dev/null +++ b/hack/k8s-cw-staging-cmri/README.md @@ -0,0 +1,54 @@ +# Deploying Clearwater into Kubernetes + +## Tables of content + +## POC + +### Architecture + +![cluster](./cluster-architect.png) + +### Kubernetes dashboard + +![dashboard](dashboard.png) + +### Ellis + +![ellis-sign](ellis-sign.png) + +![ellis-home](ellis-home.png) + +### App + +Zoiper + +![preference-accounts-general](./zoiper-preference-accounts-general.png) + +![perference-accounts-advanced](./zoiper-preference-accounts-advanced.png) + +![zoiper-preference-advanced-network](./zoiper-preference-advanced-network.png) + +![zoiper-contace](./zoiper-contact.png) + +X-Lite + +![x-lite-preference-accounts](./x-lite-preference-accounts.png) + +![x-lite-preference-accounts-general](./x-lite-preference-accounts-general.png) + +![x-lite-preference-accounts-topology](./x-lite-preference-accounts-topology.png) + +![x-lite-preference-accounts-transport](./x-lite-preference-accounts-transport.png) + +![x-lite-contact](x-lite-contact.png) + + +### Trial + +*Zoiper* dial up *X-Lite* + +![Zoiper-dialup](./zoiper-dialup.png) + +*X-Lite* dial up *Zoiper* + +![X-Lite-dialup](./x-lite-dialup.png) \ No newline at end of file diff --git a/hack/k8s-cw-staging-cmri/addr100-setup.md b/hack/k8s-cw-staging-cmri/addr100-setup.md new file mode 100755 index 0000000..e4ba8f0 --- /dev/null +++ b/hack/k8s-cw-staging-cmri/addr100-setup.md @@ -0,0 +1,370 @@ +About +======= + +有关详细的操作，见`addr99-***`前缀的文件 + +Adjust FileSystem +------------------- + + [root@centos7 ~]# vgs + VG #PV #LV #SN Attr VSize VFree + vg_data 1 3 0 wz--n- 557.37g 60.00m + + [root@centos7 ~]# lvs + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + lv_home vg_data -wi-ao---- 492.41g + lv_root vg_data -wi-ao---- 50.00g + lv_swap vg_data -wi-ao---- 14.91g + + [root@centos7 ~]# mount | grep lv_home + /dev/mapper/vg_data-lv_home on /home type ext4 (rw,relatime,seclabel,data=ordered) + + [root@centos7 ~]# lvreduce --size=92.41g vg_data/lv_home + Rounding size to boundary between physical extents: 92.41 GiB + WARNING: Reducing active and open logical volume to 92.41 GiB + THIS MAY DESTROY YOUR DATA (filesystem etc.) + Do you really want to reduce lv_home? [y/n]: y + Size of logical volume vg_data/lv_home changed from 400.00 GiB (102400 extents) to 92.41 GiB (23657 extents). + Logical volume lv_home successfully resized + + [root@centos7 ~]# lvextend --size=150g vg_data/lv_root + Size of logical volume vg_data/lv_root changed from 50.00 GiB (12800 extents) to 150.00 GiB (38400 extents). + Logical volume lv_root successfully resized + + [root@centos7 ~]# lvs + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + lv_home vg_data -wi-ao---- 92.41g + lv_root vg_data -wi-ao---- 150.00g + lv_swap vg_data -wi-ao---- 14.91g + +Update CentOS packages +------------------------- + +OS + + [root@centos7 ~]# uname -r + 3.10.0-229.el7.x86_64 + +Update + + [root@centos7 ~]# yum check-update + 已加载插件：fastestmirror, langpacks + Loading mirror speeds from cached hostfile + * base: mirrors.btte.net + * extras: ftp.sjtu.edu.cn + * updates: ftp.sjtu.edu.cn + + [root@centos7 ~]# yum update + 已加载插件：fastestmirror, langpacks + Loading mirror speeds from cached hostfile + * base: mirrors.aliyun.com + * extras: mirrors.aliyun.com + * updates: mirrors.aliyun.com + +物理网络 +-------- + +Show + + [root@centos7 ~]# ip a show + 1: lo: mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: enp17s0f0: mtu 1500 qdisc mq state UP qlen 1000 + link/ether 00:0a:f7:50:1a:f0 brd ff:ff:ff:ff:ff:ff + 3: eno2: mtu 1500 qdisc mq state UP qlen 1000 + link/ether 40:f2:e9:de:c2:f2 brd ff:ff:ff:ff:ff:ff + inet 192.168.39.100/24 brd 192.168.39.255 scope global dynamic eno2 + valid_lft 336sec preferred_lft 336sec + inet6 fe80::42f2:e9ff:fede:c2f2/64 scope link + valid_lft forever preferred_lft forever + 4: enp17s0f1: mtu 1500 qdisc mq state UP qlen 1000 + link/ether 00:0a:f7:50:1a:f2 brd ff:ff:ff:ff:ff:ff + 5: eno3: mtu 1500 qdisc mq state DOWN qlen 1000 + link/ether 40:f2:e9:de:c2:f3 brd ff:ff:ff:ff:ff:ff + 6: eno4: mtu 1500 qdisc mq state DOWN qlen 1000 + link/ether 40:f2:e9:de:c2:f4 brd ff:ff:ff:ff:ff:ff + 7: eno5: mtu 1500 qdisc mq state DOWN qlen 1000 + link/ether 40:f2:e9:de:c2:f5 brd ff:ff:ff:ff:ff:ff + 8: enp0s29u1u1u5: mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000 + link/ether 42:f2:e9:de:c2:f1 brd ff:ff:ff:ff:ff:ff + inet 169.254.95.120/24 brd 169.254.95.255 scope link dynamic enp0s29u1u1u5 + valid_lft 591sec preferred_lft 591sec + inet6 fe80::40f2:e9ff:fede:c2f1/64 scope link + valid_lft forever preferred_lft forever + + [root@centos7 ~]# cat /etc/resolv.conf + nameserver 8.8.8.8 + +关闭 firewall + + [root@centos7 ~]# systemctl stop firewalld.service + + [root@centos7 ~]# systemctl disable firewalld.service + Removed symlink /etc/systemd/system/basic.target.wants/firewalld.service. + Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service. + +工具 +------ + +Such as `wget`, `netstat`, `git` + + [root@centos7 ~]# wget + wget：未指定 URL + 用法： wget [选项]... [URL]... + + 请尝试使用“wget --help”查看更多的选项。 + + [root@centos7 ~]# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 21235/rpcbind + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 31868/sshd + tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 2701/master + tcp6 0 0 :::111 :::* LISTEN 21235/rpcbind + tcp6 0 0 :::22 :::* LISTEN 31868/sshd + tcp6 0 0 ::1:25 :::* LISTEN 2701/master + + [root@centos7 ~]# yum install -y git + +安装docker, etcd, flannel +-------------------------- + +Install from `YUM` + + [root@centos7 ~]# yum install -y docker etcd flannel + 已加载插件：fastestmirror, langpacks + Loading mirror speeds from cached hostfile + * base: mirrors.btte.net + * extras: ftp.sjtu.edu.cn + * updates: ftp.sjtu.edu.cn + + + ==================================================================================================== + Package 架构 版本 源 大小 + ==================================================================================================== + 正在安装: + docker x86_64 1.10.3-46.el7.centos.14 extras 9.5 M + etcd x86_64 2.3.7-4.el7 extras 6.5 M + flannel x86_64 0.5.3-9.el7 extras 1.7 M + 为依赖而安装: + audit-libs-python x86_64 2.4.1-5.el7 base 69 k + checkpolicy x86_64 2.1.12-6.el7 base 247 k + docker-common x86_64 1.10.3-46.el7.centos.14 extras 61 k + docker-selinux x86_64 1.10.3-46.el7.centos.14 extras 79 k + libcgroup x86_64 0.41-8.el7 base 64 k + libseccomp x86_64 2.2.1-1.el7 base 49 k + libsemanage-python x86_64 2.1.10-18.el7 base 94 k + oci-register-machine x86_64 1:0-1.8.gitaf6c129.el7 extras 1.1 M + oci-systemd-hook x86_64 1:0.1.4-4.git41491a3.el7 extras 27 k + policycoreutils-python x86_64 2.2.5-20.el7 base 435 k + python-IPy noarch 0.75-6.el7 base 32 k + setools-libs x86_64 3.3.7-46.el7 base 485 k + + 事务概要 + ==================================================================================================== + 安装 3 软件包 (+12 依赖软件包) + +安装kubernetes +--------------- + +Download `kubernetes` packages + + [root@centos7 ~]# wget -c https://github.com/kubernetes/kubernetes/releases/download/v1.3.10/kubernetes.tar.gz -b + + [root@centos7 ~]# tail wget-log + 1219850K .......... .......... .......... .......... .......... 83% 39.6K 7m20s + 1219900K .......... .......... .......... .......... .......... 83% 43.1K 7m20s + 1219950K .......... .......... .......... .......... .......... 83% 53.1K 7m20s + 1220000K .......... .......... .......... .......... .......... 83% 46.3K 7m20s + 1220050K .......... .......... .......... .......... .......... 83% 39.4K 7m21s + 1220100K .......... .......... .......... .......... .......... 83% 39.1K 7m21s + 1220150K .......... .......... .......... .......... .......... 83% 38.9K 7m21s + 1220200K .......... .......... .......... .......... .......... 83% 40.4K 7m21s + 1220250K .......... .......... .......... .......... .......... 83% 53.1K 7m21s + 1220300K .......... .......... ........ + + [root@centos7 ~]# ls + anaconda-ks.cfg kubernetes.tar.gz wget-log + + [root@centos7 ~]# tar -C /opt -zxf kubernetes.tar.gz kubernetes/server/kubernetes-server-linux-amd64.tar.gz + + [root@centos7 ~]# tar -C /opt -zxf /opt/kubernetes/server/kubernetes-server-linux-amd64.tar.gz + + [root@centos7 ~]# ln -s /opt/kubernetes/server/bin/kubectl bin/ + + [root@centos7 ~]# mkdir .kube + + [root@centos7 ~]# scp 192.168.39.99:/root/.kube/config .kube/ + root@192.168.39.99's password: + config 100% 10KB 9.9KB/s 00:00 + + [root@centos7 ~]# mkdir -p /opt/kubernetes/cacerts + + [root@centos7 ~]# mkdir -p /etc/kubernetes/cacerts + + [root@centos7 ~]# scp -r 192.168.39.99:/opt/kubernetes/cacerts /opt/kubernetes/ + root@192.168.39.99's password: + ca.crt 100% 1224 1.2KB/s 00:00 + kubecfg.crt 100% 4423 4.3KB/s 00:00 + server.cert 100% 4882 4.8KB/s 00:00 + server.key 100% 1708 1.7KB/s 00:00 + kubecfg.key 100% 1704 1.7KB/s 00:00 + + [root@centos7 ~]# cp -r /opt/kubernetes/cacerts/* /etc/kubernetes/cacerts/ + + [root@centos7 ~]# scp 192.168.39.99:/etc/kubernetes/kubeconfig /etc/kubernetes/ + root@192.168.39.99's password: + kubeconfig 100% 10KB 9.9KB/s 00:00 + +Deployment tool + + [root@centos7 ~]# scp -r root@192.168.39.99:/root/{kube,kubernetes} . + +Setup `kubelet` service + + [root@centos7 ~]# scp -r 192.168.39.99:/root/kubernetes/1.3.10/centos/systemd kubernetes/1.3.10/centos/ + root@192.168.39.99's password: + kube-apiserver.service 100% 640 0.6KB/s 00:00 + kube-controller-manager.service 100% 676 0.7KB/s 00:00 + kube-proxy.service 100% 607 0.6KB/s 00:00 + kubelet.service 100% 748 0.7KB/s 00:00 + kube-scheduler.service 100% 616 0.6KB/s 00:00 + kube-scheduler 100% 723 0.7KB/s 00:00 + kubelet 100% 1791 1.8KB/s 00:00 + kube-controller-manager 100% 934 0.9KB/s 00:00 + kube-apiserver 100% 2211 2.2KB/s 00:00 + kube-proxy 100% 893 0.9KB/s 00:00 + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/system/kubelet.service /etc/systemd/system + + [root@centos7 ~]# vi kubernetes/1.3.10/centos/sysconfig/kubelet + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/conf/kubelet /etc/sysconfig/ + + [root@centos7 ~]# systemctl enable kubelet.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /etc/systemd/system/kubelet.service. + + [root@centos7 ~]# systemctl start kubelet.service + + [root@centos7 ~]# systemctl -l status kubelet.service + ● kubelet.service - Kubernetes container mgmt daemon - kubelet + Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 18:36:28 CST; 9s ago + Docs: https://github.com/kubernetes/kubernetes + Process: 20131 ExecStartPre=/usr/bin/mkdir -p /var/log/containers (code=exited, status=0/SUCCESS) + Process: 20128 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests (code=exited, status=0/SUCCESS) + Main PID: 20135 (kubelet) + CGroup: /system.slice/kubelet.service + ├─20135 /opt/kubernetes/server/bin/kubelet --address=192.168.39.100 --allow-privileged=true --api-servers=https://192.168.39.99:6443 --cadvisor-port=4194 --cert-dir=/opt/kubernetes/cacerts --cluster-dns=10.123.240.10 --cluster-domain=cluster.local --config=/etc/kubernetes/manifests --configure-cbr0=false --container-runtime=docker --docker=unix:///var/run/docker.sock --healthz-bind-address=127.0.0.1 --healthz-port=10248 --hostname-override=192.168.39.100 --kubeconfig=/etc/kubernetes/kubeconfig --master-service-namespace=default --max-open-files=1000000 --max-pods=0 --node-ip=192.168.39.100 --non-masquerade-cidr=10.0.0.0/8 --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 --pods-per-core=0 --port=10250 --read-only-port=10255 --root-dir=/var/lib/kubelet --tls-cert-file=/opt/kubernetes/cacerts/server.cert --tls-private-key-file=/opt/kubernetes/cacerts/server.key --v=2 + └─20222 journalctl -k -f + + 11月 16 18:36:29 centos7 kubelet[20135]: E1116 18:36:29.371594 20135 manager.go:240] Registration of the rkt container factory failed: unable to communicate with Rkt api service: rkt: cannot tcp Dial rkt api service: dial tcp [::1]:15441: getsockopt: connection refused + 11月 16 18:36:29 centos7 kubelet[20135]: I1116 18:36:29.371605 20135 factory.go:54] Registering systemd factory + 11月 16 18:36:29 centos7 kubelet[20135]: I1116 18:36:29.372208 20135 factory.go:86] Registering Raw factory + 11月 16 18:36:29 centos7 kubelet[20135]: I1116 18:36:29.372819 20135 manager.go:1072] Started watching for new ooms in manager + 11月 16 18:36:29 centos7 kubelet[20135]: I1116 18:36:29.374199 20135 oomparser.go:185] oomparser using systemd + 11月 16 18:36:29 centos7 kubelet[20135]: I1116 18:36:29.375005 20135 manager.go:281] Starting recovery of all containers + 11月 16 18:36:29 centos7 kubelet[20135]: I1116 18:36:29.375814 20135 manager.go:286] Recovery completed + 11月 16 18:36:29 centos7 kubelet[20135]: I1116 18:36:29.386537 20135 kubelet.go:2924] Recording NodeReady event message for node 192.168.39.100 + 11月 16 18:36:34 centos7 kubelet[20135]: I1116 18:36:34.122062 20135 kubelet.go:2566] SyncLoop (ADD, "file"): "" + 11月 16 18:36:34 centos7 kubelet[20135]: I1116 18:36:34.122126 20135 kubelet.go:2566] SyncLoop (ADD, "api"): "" + +Setup `kube-proxy` + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/system/kube-proxy.service /etc/systemd/system + + [root@centos7 ~]# vi kubernetes/1.3.10/centos/systemd/conf/kube-proxy + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/conf/kube-proxy /etc/sysconfig/ + + [root@centos7 ~]# systemctl enable kube-proxy.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /etc/systemd/system/kube-proxy.service. + + [root@centos7 ~]# systemctl start kube-proxy.service + + [root@centos7 ~]# systemctl -l status kube-proxy.service + ● kube-proxy.service - Kubernetes proxy networking + Loaded: loaded (/etc/systemd/system/kube-proxy.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 18:39:58 CST; 10s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 20529 (kube-proxy) + CGroup: /system.slice/kube-proxy.service + └─20529 /opt/kubernetes/server/bin/kube-proxy --bind-address=192.168.39.100 --cluster-cidr=10.120.0.0/14 --healthz-bind-address=127.0.0.1 --healthz-port=10249 --hostname-override=192.168.39.100 --iptables-masquerade-bit=14 --kubeconfig=/etc/kubernetes/kubeconfig --master=https://192.168.39.99:6443 --v=2 + + 11月 16 18:39:58 centos7 kube-proxy[20529]: I1116 18:39:58.943976 20529 server.go:155] setting OOM scores is unsupported in this build + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.010762 20529 server.go:202] Using iptables Proxier. + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.014183 20529 proxier.go:216] missing br-netfilter module or unset br-nf-call-iptables; proxy may not work as intended + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.014239 20529 server.go:214] Tearing down userspace rules. + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.035077 20529 conntrack.go:40] Setting nf_conntrack_max to 1048576 + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.036283 20529 conntrack.go:57] Setting conntrack hashsize to 262144 + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.037568 20529 conntrack.go:62] Setting nf_conntrack_tcp_timeout_established to 86400 + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.037858 20529 proxier.go:440] Adding new service "default/kubernetes:https" at 10.123.240.1:443/TCP + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.038096 20529 proxier.go:674] Not syncing iptables until Services and Endpoints have been received from master + 11月 16 18:39:59 centos7 kube-proxy[20529]: I1116 18:39:59.040693 20529 proxier.go:516] Setting endpoints for "default/kubernetes:https" to [192.168.39.99:6443] + +Validation + + [root@centos7 ~]# kubectl get nodes + NAME STATUS AGE + 192.168.39.100 Ready 4m + 192.168.39.99 Ready 45m + +配置 POD networking + + [root@centos7 ~]# vi /etc/sysconfig/flanneld + + [root@centos7 ~]# systemctl enable flanneld.service + Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /usr/lib/systemd/system/flanneld.service. + Created symlink from /etc/systemd/system/docker.service.requires/flanneld.service to /usr/lib/systemd/system/flanneld.service. + + [root@centos7 ~]# ip a show dockero + Device "dockero" does not exist. + + [root@centos7 ~]# ip a show docker0 + 9: docker0: mtu 1500 qdisc noqueue state DOWN + link/ether 02:42:ff:91:b9:7d brd ff:ff:ff:ff:ff:ff + inet 172.17.0.1/16 scope global docker0 + valid_lft forever preferred_lft forever + + [root@centos7 ~]# systemctl start flanneld.service + + [root@centos7 ~]# systemctl -l status flanneld.service + ● flanneld.service - Flanneld overlay address etcd agent + Loaded: loaded (/usr/lib/systemd/system/flanneld.service; enabled; vendor preset: disabled) + Active: active (running) since 四 2016-11-17 18:07:04 CST; 7s ago + Process: 29760 ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker (code=exited, status=0/SUCCESS) + Main PID: 29749 (flanneld) + CGroup: /system.slice/flanneld.service + └─29749 /usr/bin/flanneld -etcd-endpoints=http://192.168.39.99:2379 -etcd-prefix=/cmri.com/network -iface=192.168.39.100 + + 11月 17 18:07:04 centos7 systemd[1]: Starting Flanneld overlay address etcd agent... + 11月 17 18:07:04 centos7 flanneld[29749]: I1117 18:07:04.176414 29749 main.go:275] Installing signal handlers + 11月 17 18:07:04 centos7 flanneld[29749]: I1117 18:07:04.178547 29749 main.go:188] Using 192.168.39.100 as external interface + 11月 17 18:07:04 centos7 flanneld[29749]: I1117 18:07:04.178606 29749 main.go:189] Using 192.168.39.100 as external endpoint + 11月 17 18:07:04 centos7 flanneld[29749]: I1117 18:07:04.182969 29749 etcd.go:204] Picking subnet in range 10.120.2.0 ... 10.121.254.0 + 11月 17 18:07:04 centos7 flanneld[29749]: I1117 18:07:04.184185 29749 etcd.go:84] Subnet lease acquired: 10.120.132.0/23 + 11月 17 18:07:04 centos7 flanneld[29749]: I1117 18:07:04.195642 29749 udp.go:222] Watching for new subnet leases + 11月 17 18:07:04 centos7 flanneld[29749]: I1117 18:07:04.197550 29749 udp.go:247] Subnet added: 10.120.138.0/23 + 11月 17 18:07:04 centos7 systemd[1]: Started Flanneld overlay address etcd agent. + + [root@centos7 ~]# systemctl restart docker.service + + [root@centos7 ~]# ip addr show docker0 + 9: docker0: mtu 1500 qdisc noqueue state DOWN + link/ether 02:42:ff:91:b9:7d brd ff:ff:ff:ff:ff:ff + inet 10.120.132.1/23 scope global docker0 + valid_lft forever preferred_lft forever + + [root@centos7 ~]# ping -c3 10.120.138.1 + PING 10.120.138.1 (10.120.138.1) 56(84) bytes of data. + 64 bytes from 10.120.138.1: icmp_seq=1 ttl=62 time=1.13 ms + 64 bytes from 10.120.138.1: icmp_seq=2 ttl=62 time=0.357 ms + 64 bytes from 10.120.138.1: icmp_seq=3 ttl=62 time=0.391 ms + + --- 10.120.138.1 ping statistics --- + 3 packets transmitted, 3 received, 0% packet loss, time 2001ms + rtt min/avg/max/mdev = 0.357/0.628/1.136/0.359 ms diff --git a/hack/k8s-cw-staging-cmri/addr99-setup.md b/hack/k8s-cw-staging-cmri/addr99-setup.md new file mode 100644 index 0000000..4dd7cd5 --- /dev/null +++ b/hack/k8s-cw-staging-cmri/addr99-setup.md @@ -0,0 +1,647 @@ +Table of contents +================== + +* Adjust LVM + + +Adjust lVM +------------ + +Staging的FileSystem目标是满足验证 + +* root卷需要足够的空间，以可分配给kubelet和docker, etcd + + `/var/lib/kubelet` + + `/var/lib/docker` + + `/var/lib/etcd` + +* 在production中，上述FS将使用单独的High Performance Stroage + +初始的FS + + [root@centos7 ~]# vgs + VG #PV #LV #SN Attr VSize VFree + vg_data 1 3 0 wz--n- 557.37g 60.00m + + [root@centos7 ~]# lvs + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + lv_home vg_data -wi-ao---- 492.41g + lv_root vg_data -wi-ao---- 50.00g + lv_swap vg_data -wi-ao---- 14.91g + +调整后的FS + + [root@centos7 ~]# lvreduce --size=-400g vg_data/lv_home + WARNING: Reducing active and open logical volume to 92.41 GiB + THIS MAY DESTROY YOUR DATA (filesystem etc.) + Do you really want to reduce lv_home? [y/n]: y + Size of logical volume vg_data/lv_home changed from 492.41 GiB (126056 extents) to 92.41 GiB (23656 extents). + Logical volume lv_home successfully resized + + [root@centos7 ~]# lvs + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + lv_home vg_data -wi-ao---- 92.41g + lv_root vg_data -wi-ao---- 50.00g + lv_swap vg_data -wi-ao---- 14.91g + + [root@centos7 ~]# lvextend --size=+100g vg_data/lv_root + Size of logical volume vg_data/lv_root changed from 50.00 GiB (12800 extents) to 150.00 GiB (38400 extents). + Logical volume lv_root successfully resized + + [root@centos7 ~]# lvs + LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert + lv_home vg_data -wi-ao---- 92.41g + lv_root vg_data -wi-ao---- 150.00g + lv_swap vg_data -wi-ao---- 14.91g + +在pool中的未分配storage + +* vg_data中未分配：300G + +* /dev/sdb，/dev/sdc + + +Update CentOS packages +----------------------- + +系统摘要 + + [root@centos7 ~]# cat /etc/redhat-release + CentOS Linux release 7.1.1503 (Core) + + [root@centos7 ~]# uname -r + 3.10.0-229.el7.x86_64 + +Update + + [root@centos7 ~]# yum check-update + + [root@centos7 ~]# yum update + +Physical Networking +-------------------- + +检查 + + [root@centos7 ~]# ip a + 1: lo: mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: enp17s0f0: mtu 1500 qdisc mq state UP qlen 1000 + link/ether 00:0a:f7:50:1f:70 brd ff:ff:ff:ff:ff:ff + 3: eno2: mtu 1500 qdisc mq state UP qlen 1000 + link/ether 40:f2:e9:de:30:e2 brd ff:ff:ff:ff:ff:ff + inet 192.168.39.99/24 brd 192.168.39.255 scope global dynamic eno2 + valid_lft 588sec preferred_lft 588sec + inet6 fe80::42f2:e9ff:fede:30e2/64 scope link + valid_lft forever preferred_lft forever + 4: enp17s0f1: mtu 1500 qdisc mq state DOWN qlen 1000 + link/ether 00:0a:f7:50:1f:72 brd ff:ff:ff:ff:ff:ff + 5: eno3: mtu 1500 qdisc mq state DOWN qlen 1000 + link/ether 40:f2:e9:de:30:e3 brd ff:ff:ff:ff:ff:ff + 6: eno4: mtu 1500 qdisc mq state DOWN qlen 1000 + link/ether 40:f2:e9:de:30:e4 brd ff:ff:ff:ff:ff:ff + 7: eno5: mtu 1500 qdisc mq state DOWN qlen 1000 + link/ether 40:f2:e9:de:30:e5 brd ff:ff:ff:ff:ff:ff + 9: enp0s29u1u1u5: mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether 42:f2:e9:de:30:e1 brd ff:ff:ff:ff:ff:ff + inet 169.254.95.120/24 brd 169.254.95.255 scope link dynamic enp0s29u1u1u5 + valid_lft 461sec preferred_lft 461sec + inet6 fe80::40f2:e9ff:fede:30e1/64 scope link + valid_lft forever preferred_lft forever + + [root@centos7 ~]# systemctl is-active NetworkManager.service + active + + [root@centos7 ~]# nmcli d + 警告：nmcli (1.0.6) 和 NetworkManager (1.0.0) 版本不匹配。请使用 --nocheck 抑制警告。 + 设备 类型 状态 CONNECTION + eno2 ethernet 连接的 eno2 + enp0s29u1u1u5 ethernet 连接的 enp0s29u1u1u5 + enp17s0f0 ethernet 已断开 -- + eno3 ethernet 不可用 -- + eno4 ethernet 不可用 -- + eno5 ethernet 不可用 -- + enp17s0f1 ethernet 不可用 -- + lo loopback 未管理 -- + + [root@centos7 ~]# nmcli c + 警告：nmcli (1.0.6) 和 NetworkManager (1.0.0) 版本不匹配。请使用 --nocheck 抑制警告。 + 名称 UUID 类型 设备 + enp0s29u1u1u5 e6d1136b-2c31-4690-a147-98cb2d9290b9 802-3-ethernet enp0s29u1u1u5 + eno2 2f96bfde-c41a-47bc-bd50-e5e32ca55ab7 802-3-ethernet eno2 + eno3 7dc754bc-e237-47f4-89b3-19767fedd636 802-3-ethernet -- + Wired connection 1 06a7ad6b-32d9-4346-9370-ba28d5452426 802-3-ethernet enp17s0f0 + eno5 0bbbc041-6935-4006-85bd-a71a1fd20e3c 802-3-ethernet -- + enp17s0f1 a0d554e7-da72-47c3-a2c5-3578046376b0 802-3-ethernet -- + eno4 cd887cee-a45a-41a4-88e5-ac897ebfc945 802-3-ethernet -- + + [root@centos7 ~]# cat /etc/resolv.conf + nameserver 8.8.8.8 + +关闭firewall + + [root@centos7 ~]# systemctl -l status firewalld.service + ● firewalld.service - firewalld - dynamic firewall daemon + Loaded: loaded (/usr/lib/systemd/system/firewalld.service; enabled; vendor preset: enabled) + Active: active (running) since 五 2016-11-11 16:45:58 CST; 5 days ago + Main PID: 32729 (firewalld) + CGroup: /system.slice/firewalld.service + └─32729 /usr/bin/python -Es /usr/sbin/firewalld --nofork --nopid + + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t nat -C POSTROUTING -s 172.17.0.0/16 ! -o docker0 -j MASQUERADE' failed: iptables: No chain/target/match by that name. + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t nat -C DOCKER -i docker0 -j RETURN' failed: iptables: Bad rule (does a matching rule exist in that chain?). + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -D FORWARD -i docker0 -o docker0 -j DROP' failed: iptables: Bad rule (does a matching rule exist in that chain?). + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t filter -C FORWARD -i docker0 -o docker0 -j ACCEPT' failed: iptables: Bad rule (does a matching rule exist in that chain?). + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t filter -C FORWARD -i docker0 ! -o docker0 -j ACCEPT' failed: iptables: Bad rule (does a matching rule exist in that chain?). + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t filter -C FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT' failed: iptables: Bad rule (does a matching rule exist in that chain?). + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t nat -C PREROUTING -m addrtype --dst-type LOCAL -j DOCKER' failed: iptables: No chain/target/match by that name. + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t nat -C OUTPUT -m addrtype --dst-type LOCAL -j DOCKER ! --dst 127.0.0.0/8' failed: iptables: No chain/target/match by that name. + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t filter -C FORWARD -o docker0 -j DOCKER' failed: iptables: No chain/target/match by that name. + 11月 16 17:55:50 centos7 firewalld[32729]: 2016-11-16 17:55:50 ERROR: COMMAND_FAILED: '/sbin/iptables -w2 -t filter -C FORWARD -j DOCKER-ISOLATION' failed: iptables: No chain/target/match by that name. + [root@centos7 ~]# systemctl stop firewalld.service + + [root@centos7 ~]# systemctl disable firewalld.service + Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service. + Removed symlink /etc/systemd/system/basic.target.wants/firewalld.service. + +安装软件包docker, etcd, flannel +------------------------------- + +Install `docker` packages + + [root@centos7 ~]# yum install -y docker + 已加载插件：fastestmirror, langpacks + Loading mirror speeds from cached hostfile + * base: mirrors.aliyun.com + * extras: mirrors.aliyun.com + * updates: mirrors.aliyun.com + + + ==================================================================================================== + Package 架构 版本 源 大小 + ==================================================================================================== + 正在安装: + docker x86_64 1.10.3-46.el7.centos.14 extras 9.5 M + 为依赖而安装: + audit-libs-python x86_64 2.4.1-5.el7 base 69 k + checkpolicy x86_64 2.1.12-6.el7 base 247 k + docker-common x86_64 1.10.3-46.el7.centos.14 extras 61 k + docker-selinux x86_64 1.10.3-46.el7.centos.14 extras 79 k + libcgroup x86_64 0.41-8.el7 base 64 k + libseccomp x86_64 2.2.1-1.el7 base 49 k + libsemanage-python x86_64 2.1.10-18.el7 base 94 k + oci-register-machine x86_64 1:0-1.8.gitaf6c129.el7 extras 1.1 M + oci-systemd-hook x86_64 1:0.1.4-4.git41491a3.el7 extras 27 k + policycoreutils-python x86_64 2.2.5-20.el7 base 435 k + python-IPy noarch 0.75-6.el7 base 32 k + setools-libs x86_64 3.3.7-46.el7 base 485 k + + 事务概要 + ==================================================================================================== + 安装 1 软件包 (+12 依赖软件包) + +Install `etcd` and `flannel` packages + + [root@centos7 ~]# yum install -y etcd flannel + 已加载插件：fastestmirror, langpacks + Loading mirror speeds from cached hostfile + * base: mirrors.aliyun.com + * extras: mirrors.aliyun.com + * updates: mirrors.aliyun.com + 正在解决依赖关系 + --> 正在检查事务 + ---> 软件包 etcd.x86_64.0.2.3.7-4.el7 将被 安装 + ---> 软件包 flannel.x86_64.0.0.5.3-9.el7 将被 安装 + --> 解决依赖关系完成 + + 依赖关系解决 + + ==================================================================================================== + Package 架构 版本 源 大小 + ==================================================================================================== + 正在安装: + etcd x86_64 2.3.7-4.el7 extras 6.5 M + flannel x86_64 0.5.3-9.el7 extras 1.7 M + + 事务概要 + ==================================================================================================== + 安装 2 软件包 + + + [root@centos7 ~]# sudo systemctl list-unit-files docker.service etcd.service flanneld.service + UNIT FILE STATE + docker.service disabled + etcd.service disabled + flanneld.service disabled + + 3 unit files listed. + +Setup `etcd` service + + [root@centos7 ~]# vi /etc/etcd/etcd.conf + ETCD_NAME=k8s-cw-project + ETCD_DATA_DIR="/var/lib/etcd/k8s-cw-project.etcd" + ETCD_LISTEN_PEER_URLS="http://192.168.39.99:2380" + ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.39.99:2380" + ETCD_INITIAL_CLUSTER="k8s-cw-project=http://192.168.39.99:2380" + ETCD_ADVERTISE_CLIENT_URLS="http://192.168.39.99:2379" + + + [root@centos7 ~]# systemctl enable etcd.service + Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service. + [root@centos7 ~]# systemctl start etcd.service + + [root@centos7 ~]# systemctl -l status etcd.service + ● etcd.service - Etcd Server + Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 16:37:10 CST; 55s ago + Main PID: 11026 (etcd) + CGroup: /system.slice/etcd.service + └─11026 /usr/bin/etcd --name=k8s-cw-project --data-dir=/var/lib/etcd/k8s-cw-project.etcd --listen-client-urls=http://0.0.0.0:2379 + + [root@centos7 ~]# ls /var/lib/etcd + k8s-cw-project.etcd + + [root@centos7 ~]# curl http://192.168.39.99:2379/version + {"etcdserver":"2.3.7","etcdcluster":"2.3.0"} + + [root@centos7 ~]# sudo netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 192.168.39.99:2380 0.0.0.0:* LISTEN 11026/etcd + tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 22576/rpcbind + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 741/sshd + tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 2762/master + tcp6 0 0 :::2379 :::* LISTEN 11026/etcd + tcp6 0 0 :::111 :::* LISTEN 22576/rpcbind + tcp6 0 0 :::22 :::* LISTEN 741/sshd + tcp6 0 0 ::1:25 :::* LISTEN 2762/master + +安装kubernetes +--------------- + +Download `kubernetes` packages + + fanhonglingdeMacBook-Pro:trusty64 fanhongling$ scp -r /Volumes/TOURO\ Mobile/kubernetes/{kube,kubernetes,kubernetes.tar.gz} root@192.168.39.99:/root + + [root@centos7 ~]# tar -C /opt -zxf kubernetes.tar.gz kubernetes/server/kubernetes-server-linux-amd64.tar.gz + [root@centos7 ~]# tar -C /opt -zxf /opt/kubernetes/server/kubernetes-server-linux-amd64.tar.gz + + [root@centos7 ~]# /opt/kubernetes/server/bin/kubelet --version + Kubernetes v1.3.10 + +Generate CA and certs + + [root@centos7 ~]# vi salt-make-ca-cert.sh + + root@centos7 ~]# ./salt-make-ca-cert.sh + IP:192.168.39.99,IP:10.123.240.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local + Generate ca certs into /root/.kube/cacerts + +Setup `kube-apiserver` servie + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/system/kube-apiserver.service /etc/systemd/system + + [root@centos7 ~]# vi kubernetes/1.3.10/centos/systemd/conf/kube-apiserver + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/conf/kube-apiserver /etc/sysconfig/ + + [root@centos7 ~]# systemctl enable kube-apiserver.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /etc/systemd/system/kube-apiserver.service. + [root@centos7 ~]# systemctl start kube-apiserver.service + [root@centos7 ~]# systemctl -l status kube-apiserver.service + ● kube-apiserver.service - Kubernetes API server + Loaded: loaded (/etc/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 16:39:36 CST; 7s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 11113 (kube-apiserver) + CGroup: /system.slice/kube-apiserver.service + └─11113 /opt/kubernetes/server/bin/kube-apiserver --admission-control=AlwaysAdmit --advertise-address=192.168.39.99 --allow-privileged=true --apiserver-count=1 --bind-address=0.0.0.0 --cert-dir=/opt/kubernetes/cacerts --client-ca-file=/opt/kubernetes/cacerts/ca.crt --enable-garbage-collector=false --enable-swagger-ui=false --etcd-servers=http://192.168.39.99:2379 --insecure-bind-address=127.0.0.1 --insecure-port=8080 --kubelet-certificate-authority=/opt/kubernetes/cacerts/ca.crt --kubelet-client-certificate=/opt/kubernetes/cacerts/kubecfg.crt --kubelet-client-key=/opt/kubernetes/cacerts/kubecfg.key --kubelet-https=true --master-service-namespace=default --runtime-config=api/all=true --secure-port=6443 --service-account-key-file=/opt/kubernetes/cacerts/server.key --service-cluster-ip-range=10.123.240.0/20 --tls-cert-file=/opt/kubernetes/cacerts/server.cert --tls-private-key-file=/opt/kubernetes/cacerts/server.key --v=2 + + 11月 16 16:39:36 centos7 systemd[1]: Starting Kubernetes API server... + 11月 16 16:39:36 centos7 kube-apiserver[11113]: I1116 16:39:36.163537 11113 genericapiserver.go:606] Will report 192.168.39.99 as public IP address. + 11月 16 16:39:36 centos7 kube-apiserver[11113]: I1116 16:39:36.168134 11113 genericapiserver.go:288] Node port range unspecified. Defaulting to 30000-32767. + 11月 16 16:39:36 centos7 kube-apiserver[11113]: W1116 16:39:36.240952 11113 controller.go:307] Resetting endpoints for master service "kubernetes" to kind:"" apiVersion:"" + 11月 16 16:39:36 centos7 kube-apiserver[11113]: [restful] 2016/11/16 16:39:36 log.go:30: [restful/swagger] listing is available at https://192.168.39.99:6443/swaggerapi/ + 11月 16 16:39:36 centos7 kube-apiserver[11113]: [restful] 2016/11/16 16:39:36 log.go:30: [restful/swagger] https://192.168.39.99:6443/swaggerui/ is mapped to folder /swagger-ui/ + 11月 16 16:39:36 centos7 kube-apiserver[11113]: I1116 16:39:36.268253 11113 genericapiserver.go:690] Serving securely on 0.0.0.0:6443 + 11月 16 16:39:36 centos7 kube-apiserver[11113]: I1116 16:39:36.268276 11113 genericapiserver.go:734] Serving insecurely on 127.0.0.1:8080 + 11月 16 16:39:36 centos7 systemd[1]: Started Kubernetes API server. + + [root@centos7 ~]# sudo netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 192.168.39.99:2380 0.0.0.0:* LISTEN 11026/etcd + tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 22576/rpcbind + tcp 0 0 127.0.0.1:8080 0.0.0.0:* LISTEN 11113/kube-apiserve + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 741/sshd + tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 2762/master + tcp6 0 0 :::6443 :::* LISTEN 11113/kube-apiserve + tcp6 0 0 :::2379 :::* LISTEN 11026/etcd + tcp6 0 0 :::111 :::* LISTEN 22576/rpcbind + tcp6 0 0 :::22 :::* LISTEN 741/sshd + tcp6 0 0 ::1:25 :::* LISTEN 2762/master + + [root@centos7 ~]# etcdctl ls + /registry + + [root@centos7 ~]# etcdctl ls --recursive /registry + /registry/ranges + /registry/ranges/serviceips + /registry/ranges/servicenodeports + /registry/namespaces + /registry/namespaces/default + /registry/namespaces/kube-system + /registry/services + /registry/services/specs + /registry/services/specs/default + /registry/services/specs/default/kubernetes + /registry/services/endpoints + /registry/services/endpoints/default + /registry/services/endpoints/default/kubernetes + + [root@centos7 ~]# curl http://127.0.0.1:8080 + { + "paths": [ + "/api", + "/api/v1", + "/apis", + "/apis/apps", + "/apis/apps/v1alpha1", + "/apis/autoscaling", + "/apis/autoscaling/v1", + "/apis/batch", + "/apis/batch/v1", + "/apis/batch/v2alpha1", + "/apis/extensions", + "/apis/extensions/v1beta1", + "/apis/policy", + "/apis/policy/v1alpha1", + "/apis/rbac.authorization.k8s.io", + "/apis/rbac.authorization.k8s.io/v1alpha1", + "/healthz", + "/healthz/ping", + "/logs/", + "/metrics", + "/swaggerapi/", + "/ui/", + "/version" + ] + } + +Setup `kubectl` tool + + [root@centos7 ~]# mkdir bin + + [root@centos7 ~]# ln -s /opt/kubernetes/server/bin/kubectl bin/ + + [root@centos7 ~]# kubectl config set-cluster kube --server=https://192.168.39.99:6443 + cluster "kube" set. + + [root@centos7 ~]# encoded=$(sudo base64 -w0 /opt/kubernetes/cacerts/ca.crt); kubectl config set clusters.kube.certificate-authority-data "$encoded" + property "clusters.kube.certificate-authority-data" set. + + [root@centos7 ~]# kubectl config set-credentials admin + user "admin" set. + + [root@centos7 ~]# encoded=$(sudo base64 -w0 /opt/kubernetes/cacerts/kubecfg.crt); kubectl config set users.admin.client-certificate-data "$encoded" + property "users.admin.client-certificate-data" set. + + [root@centos7 ~]# encoded=$(sudo base64 -w0 /opt/kubernetes/cacerts/kubecfg.key); kubectl config set users.admin.client-key-data "$encoded" + property "users.admin.client-key-data" set. + + [root@centos7 ~]# kubectl config set-context kube-admin --cluster=kube --user=admin + context "kube-admin" set. + + [root@centos7 ~]# kubectl version + Client Version: version.Info{Major:"1", Minor:"3", GitVersion:"v1.3.10", GitCommit:"c3e367ec9eae7338ac4e2a57f293634891319b7c", GitTreeState:"clean", BuildDate:"2016-10-31T21:25:43Z", GoVersion:"go1.6.2", Compiler:"gc", Platform:"linux/amd64"} + Server Version: version.Info{Major:"1", Minor:"3", GitVersion:"v1.3.10", GitCommit:"c3e367ec9eae7338ac4e2a57f293634891319b7c", GitTreeState:"clean", BuildDate:"2016-10-31T21:06:41Z", GoVersion:"go1.6.2", Compiler:"gc", Platform:"linux/amd64"} + + [root@centos7 ~]# kubectl api-versions + apps/v1alpha1 + autoscaling/v1 + batch/v1 + batch/v2alpha1 + extensions/v1beta1 + policy/v1alpha1 + rbac.authorization.k8s.io/v1alpha1 + v1 + +Setup `kube-controller-manager` service + + [root@centos7 ~]# mkdir /etc/kubernetes + + [root@centos7 ~]# cp ~/.kube/config /etc/kubernetes/kubeconfig + + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/system/kube-controller-manager.service /etc/systemd/system + [root@centos7 ~]# vi kubernetes/1.3.10/centos/systemd/conf/kube-controller-manager + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/conf/kube-controller-manager /etc/sysconfig/ + [root@centos7 ~]# systemctl enable kube-controller-manager + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /etc/systemd/system/kube-controller-manager.service. + [root@centos7 ~]# systemctl start kube-controller-manager + [root@centos7 ~]# systemctl -l status kube-controller-manager + + + [root@centos7 ~]# systemctl -l status kube-controller-manager + ● kube-controller-manager.service - Kubernetes controller manager + Loaded: loaded (/etc/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 16:57:31 CST; 10s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 12095 (kube-controller) + CGroup: /system.slice/kube-controller-manager.service + └─12095 /opt/kubernetes/server/bin/kube-controller-manager --address=192.168.39.99 --cluster-cidr=10.120.0.0/14 --cluster-name=kubernetes --enable-garbage-collector=false --kubeconfig=/etc/kubernetes/kubeconfig --master=https://192.168.39.99:6443 --port=10252 --root-ca-file=/opt/kubernetes/cacerts/ca.crt --service-account-private-key-file=/opt/kubernetes/cacerts/server.key --service-cluster-ip-range=10.123.240.0/20 --v=2 + +Setup `kube-scheduler` service + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/system/kube-scheduler.service /etc/systemd/system + + [root@centos7 ~]# vim kubernetes/1.3.10/centos/systemd/conf/kube-scheduler + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/conf/kube-scheduler /etc/sysconfig/ + + [root@centos7 ~]# systemctl enable kube-scheduler.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /etc/systemd/system/kube-scheduler.service. + + [root@centos7 ~]# systemctl start kube-scheduler.service + + [root@centos7 ~]# systemctl -l status kube-scheduler.service + ● kube-scheduler.service - Kubernetes scheduler + Loaded: loaded (/etc/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 17:14:37 CST; 7s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 12651 (kube-scheduler) + CGroup: /system.slice/kube-scheduler.service + └─12651 /opt/kubernetes/server/bin/kube-scheduler --address=192.168.39.99 --algorithm-provider=DefaultProvider --kubeconfig=/etc/kubernetes/kubeconfig --master=https://192.168.39.99:6443 --port=10251 --scheduler-name=default-scheduler --v=2 + + 11月 16 17:14:37 centos7 systemd[1]: Started Kubernetes scheduler. + 11月 16 17:14:37 centos7 systemd[1]: Starting Kubernetes scheduler... + 11月 16 17:14:37 centos7 kube-scheduler[12651]: I1116 17:14:37.371217 12651 factory.go:255] Creating scheduler from algorithm provider 'DefaultProvider' + 11月 16 17:14:37 centos7 kube-scheduler[12651]: I1116 17:14:37.371443 12651 factory.go:301] creating scheduler with fit predicates 'map[NoDiskConflict:{} NoVolumeZoneConflict:{} MaxEBSVolumeCount:{} MaxGCEPDVolumeCount:{} GeneralPredicates:{} PodToleratesNodeTaints:{} CheckNodeMemoryPressure:{}]' and priority functions 'map[BalancedResourceAllocation:{} SelectorSpreadPriority:{} NodeAffinityPriority:{} TaintTolerationPriority:{} LeastRequestedPriority:{}] + +Setup `kubelet` service + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/system/kubelet.service /etc/systemd/system + + [root@centos7 ~]# vi kubernetes/1.3.10/centos/systemd/conf/kubelet + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/conf/kubelet /etc/sysconfig/ + + [root@centos7 ~]# systemctl start kubelet.service + + [root@centos7 ~]# systemctl -l status kubelet.service + ● kubelet.service - Kubernetes container mgmt daemon - kubelet + Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 17:55:50 CST; 9s ago + Docs: https://github.com/kubernetes/kubernetes + Process: 14085 ExecStartPre=/usr/bin/mkdir -p /var/log/containers (code=exited, status=0/SUCCESS) + Process: 14080 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests (code=exited, status=0/SUCCESS) + Main PID: 14090 (kubelet) + CGroup: /system.slice/kubelet.service + ├─14090 /opt/kubernetes/server/bin/kubelet --address=192.168.39.99 --allow-privileged=true --api-servers=https://192.168.39.99:6443 --cadvisor-port=4194 --cert-dir=/opt/kubernetes/cacerts --cluster-dns=10.123.240.10 --cluster-domain=cluster.local --config=/etc/kubernetes/manifests --configure-cbr0=false --container-runtime=docker --docker=unix:///var/run/docker.sock --healthz-bind-address=127.0.0.1 --healthz-port=10248 --hostname-override=192.168.39.99 --kubeconfig=/etc/kubernetes/kubeconfig --master-service-namespace=default --max-open-files=1000000 --max-pods=0 --node-ip=192.168.39.99 --non-masquerade-cidr=10.0.0.0/8 --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 --pods-per-core=0 --port=10250 --read-only-port=10255 --root-dir=/var/lib/kubelet --tls-cert-file=/opt/kubernetes/cacerts/server.cert --tls-private-key-file=/opt/kubernetes/cacerts/server.key --v=2 + └─14173 journalctl -k -f + + 11月 16 17:55:51 centos7 kubelet[14090]: E1116 17:55:51.869844 14090 manager.go:240] Registration of the rkt container factory failed: unable to communicate with Rkt api service: rkt: cannot tcp Dial rkt api service: dial tcp [::1]:15441: getsockopt: connection refused + 11月 16 17:55:51 centos7 kubelet[14090]: I1116 17:55:51.869854 14090 factory.go:54] Registering systemd factory + 11月 16 17:55:51 centos7 kubelet[14090]: I1116 17:55:51.870428 14090 factory.go:86] Registering Raw factory + 11月 16 17:55:51 centos7 kubelet[14090]: I1116 17:55:51.871001 14090 manager.go:1072] Started watching for new ooms in manager + 11月 16 17:55:51 centos7 kubelet[14090]: I1116 17:55:51.872466 14090 oomparser.go:185] oomparser using systemd + 11月 16 17:55:51 centos7 kubelet[14090]: I1116 17:55:51.873417 14090 manager.go:281] Starting recovery of all containers + 11月 16 17:55:51 centos7 kubelet[14090]: I1116 17:55:51.874231 14090 manager.go:286] Recovery completed + 11月 16 17:55:51 centos7 kubelet[14090]: I1116 17:55:51.893660 14090 kubelet.go:2924] Recording NodeReady event message for node 192.168.39.99 + 11月 16 17:55:56 centos7 kubelet[14090]: I1116 17:55:56.626543 14090 kubelet.go:2566] SyncLoop (ADD, "file"): "" + 11月 16 17:55:56 centos7 kubelet[14090]: I1116 17:55:56.626609 14090 kubelet.go:2566] SyncLoop (ADD, "api"): "" + +Setup `kube-proxy` service + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/system/kube-proxy.service /etc/systemd/system + + [root@centos7 ~]# vi kubernetes/1.3.10/centos/systemd/conf/kube-proxy + + [root@centos7 ~]# cp kubernetes/1.3.10/centos/systemd/conf/kube-proxy /etc/sysconfig/ + + [root@centos7 ~]# systemctl enable kube-proxy.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /etc/systemd/system/kube-proxy.service. + + [root@centos7 ~]# systemctl start kube-proxy.service + + [root@centos7 ~]# systemctl -l status kube-proxy.service + ● kube-proxy.service - Kubernetes proxy networking + Loaded: loaded (/etc/systemd/system/kube-proxy.service; enabled; vendor preset: disabled) + Active: active (running) since 三 2016-11-16 17:59:21 CST; 6s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 14464 (kube-proxy) + CGroup: /system.slice/kube-proxy.service + └─14464 /opt/kubernetes/server/bin/kube-proxy --bind-address=192.168.39.99 --cluster-cidr=10.120.0.0/14 --healthz-bind-address=127.0.0.1 --healthz-port=10249 --hostname-override=192.168.39.99 --iptables-masquerade-bit=14 --kubeconfig=/etc/kubernetes/kubeconfig --master=https://192.168.39.99:6443 --v=2 + + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.474030 14464 server.go:155] setting OOM scores is unsupported in this build + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.548175 14464 server.go:202] Using iptables Proxier. + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.550990 14464 proxier.go:216] missing br-netfilter module or unset br-nf-call-iptables; proxy may not work as intended + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.551040 14464 server.go:214] Tearing down userspace rules. + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.570054 14464 conntrack.go:40] Setting nf_conntrack_max to 1048576 + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.571014 14464 conntrack.go:57] Setting conntrack hashsize to 262144 + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.572178 14464 conntrack.go:62] Setting nf_conntrack_tcp_timeout_established to 86400 + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.572726 14464 proxier.go:440] Adding new service "default/kubernetes:https" at 10.123.240.1:443/TCP + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.572950 14464 proxier.go:674] Not syncing iptables until Services and Endpoints have been received from master + 11月 16 17:59:21 centos7 kube-proxy[14464]: I1116 17:59:21.575306 14464 proxier.go:516] Setting endpoints for "default/kubernetes:https" to [192.168.39.99:6443] + +Validation + + [root@centos7 ~]# kubectl get nodes + NAME STATUS AGE + 192.168.39.99 Ready 4m + +配置 containers networking +-------------------------- + + [root@centos7 ~]# etcdctl set /cmri.com/network/config '{ + > "Network": "10.120.0.0/15", + > "SubnetLen": 23, + > "SubnetMax": "10.121.254.0", + > "Backend": { + > "Type": "udp" + > } + > }' + { + "Network": "10.120.0.0/15", + "SubnetLen": 23, + "SubnetMax": "10.121.254.0", + "Backend": { + "Type": "udp" + } + } + + [root@centos7 ~]# vi /etc/sysconfig/flanneld + + [root@centos7 ~]# vi /etc/sysconfig/flanneld + + [root@centos7 ~]# systemctl enable flanneld.service + Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /usr/lib/systemd/system/flanneld.service. + Created symlink from /etc/systemd/system/docker.service.requires/flanneld.service to /usr/lib/systemd/system/flanneld.service. + + [root@centos7 ~]# systemctl start flanneld.service + + [root@centos7 ~]# systemctl -l status flanneld.service + ● flanneld.service - Flanneld overlay address etcd agent + Loaded: loaded (/usr/lib/systemd/system/flanneld.service; enabled; vendor preset: disabled) + Active: active (running) since 四 2016-11-17 17:34:01 CST; 9s ago + Process: 22279 ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker (code=exited, status=0/SUCCESS) + Main PID: 22265 (flanneld) + CGroup: /system.slice/flanneld.service + └─22265 /usr/bin/flanneld -etcd-endpoints=http://192.168.39.99:2379 -etcd-prefix=/cmri.com/network + + 11月 17 17:34:01 centos7 systemd[1]: Starting Flanneld overlay address etcd agent... + 11月 17 17:34:01 centos7 flanneld[22265]: I1117 17:34:01.622452 22265 main.go:275] Installing signal handlers + 11月 17 17:34:01 centos7 flanneld[22265]: I1117 17:34:01.622617 22265 main.go:130] Determining IP address of default interface + 11月 17 17:34:01 centos7 flanneld[22265]: I1117 17:34:01.623969 22265 main.go:188] Using 192.168.39.99 as external interface + 11月 17 17:34:01 centos7 flanneld[22265]: I1117 17:34:01.624003 22265 main.go:189] Using 192.168.39.99 as external endpoint + 11月 17 17:34:01 centos7 flanneld[22265]: I1117 17:34:01.627055 22265 etcd.go:204] Picking subnet in range 10.120.2.0 ... 10.121.254.0 + 11月 17 17:34:01 centos7 flanneld[22265]: I1117 17:34:01.628088 22265 etcd.go:84] Subnet lease acquired: 10.120.138.0/23 + 11月 17 17:34:01 centos7 flanneld[22265]: I1117 17:34:01.645257 22265 udp.go:222] Watching for new subnet leases + 11月 17 17:34:01 centos7 systemd[1]: Started Flanneld overlay address etcd agent. + + [root@centos7 ~]# cat /var/run/flannel/subnet.env + FLANNEL_NETWORK=10.120.0.0/15 + FLANNEL_SUBNET=10.120.138.1/23 + FLANNEL_MTU=1472 + FLANNEL_IPMASQ=false + + [root@centos7 ~]# sudo systemctl enable docker.service + Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service. + [root@centos7 ~]# sudo systemctl restart docker.service + [root@centos7 ~]# sudo systemctl -l status docker.service + ● docker.service - Docker Application Container Engine + Loaded: loaded (/usr/lib/systemd/system/docker.service; enabled; vendor preset: disabled) + Drop-In: /usr/lib/systemd/system/docker.service.d + └─flannel.conf + Active: active (running) since 四 2016-11-17 17:36:16 CST; 9s ago + Docs: http://docs.docker.com + Main PID: 22540 (docker-current) + CGroup: /system.slice/docker.service + └─22540 /usr/bin/docker-current daemon --exec-opt native.cgroupdriver=systemd --selinux-enabled --log-driver=journald --storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=/dev/mapper/vg_data-docker--pool --storage-opt dm.use_deferred_removal=true --storage-opt dm.use_deferred_deletion=true --bip=10.120.138.1/23 --ip-masq=true --mtu=1472 + + 11月 17 17:36:16 centos7 systemd[1]: Starting Docker Application Container Engine... + 11月 17 17:36:16 centos7 docker-current[22540]: time="2016-11-17T17:36:16.419642040+08:00" level=info msg="Graph migration to content-addressability took 0.00 seconds" + 11月 17 17:36:16 centos7 docker-current[22540]: time="2016-11-17T17:36:16.434601879+08:00" level=info msg="Firewalld running: false" + 11月 17 17:36:16 centos7 docker-current[22540]: time="2016-11-17T17:36:16.573898188+08:00" level=info msg="Loading containers: start." + 11月 17 17:36:16 centos7 docker-current[22540]: time="2016-11-17T17:36:16.573995225+08:00" level=info msg="Loading containers: done." + 11月 17 17:36:16 centos7 docker-current[22540]: time="2016-11-17T17:36:16.574014427+08:00" level=info msg="Daemon has completed initialization" + 11月 17 17:36:16 centos7 docker-current[22540]: time="2016-11-17T17:36:16.574038287+08:00" level=info msg="Docker daemon" commit=cb079f6-unsupported execdriver=native-0.2 graphdriver=devicemapper version=1.10.3 + 11月 17 17:36:16 centos7 systemd[1]: Started Docker Application Container Engine. + 11月 17 17:36:16 centos7 docker-current[22540]: time="2016-11-17T17:36:16.581244678+08:00" level=info msg="API listen on /var/run/docker.sock" + + [root@centos7 ~]# sudo ip show docker0 + Object "show" is unknown, try "ip help". + + [root@centos7 ~]# sudo ip addr show docker0 + 10: docker0: mtu 1500 qdisc noqueue state DOWN + link/ether 02:42:90:66:7e:d4 brd ff:ff:ff:ff:ff:ff + inet 10.120.138.1/23 scope global docker0 + valid_lft forever preferred_lft forever diff --git a/hack/k8s-cw-staging-cmri/cluster-architect.png b/hack/k8s-cw-staging-cmri/cluster-architect.png new file mode 100755 index 0000000..900ac9d Binary files /dev/null and b/hack/k8s-cw-staging-cmri/cluster-architect.png differ diff --git a/hack/k8s-cw-staging-cmri/dashboard.png b/hack/k8s-cw-staging-cmri/dashboard.png new file mode 100644 index 0000000..07090e3 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/dashboard.png differ diff --git a/hack/k8s-cw-staging-cmri/ellis-home.png b/hack/k8s-cw-staging-cmri/ellis-home.png new file mode 100644 index 0000000..6646310 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/ellis-home.png differ diff --git a/hack/k8s-cw-staging-cmri/ellis-sign.png b/hack/k8s-cw-staging-cmri/ellis-sign.png new file mode 100644 index 0000000..5b3cdc3 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/ellis-sign.png differ diff --git a/hack/k8s-cw-staging-cmri/x-lite-contact.png b/hack/k8s-cw-staging-cmri/x-lite-contact.png new file mode 100644 index 0000000..52d7674 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/x-lite-contact.png differ diff --git a/hack/k8s-cw-staging-cmri/x-lite-dialup.png b/hack/k8s-cw-staging-cmri/x-lite-dialup.png new file mode 100644 index 0000000..cb1215e Binary files /dev/null and b/hack/k8s-cw-staging-cmri/x-lite-dialup.png differ diff --git a/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-general.png b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-general.png new file mode 100644 index 0000000..dc171ca Binary files /dev/null and b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-general.png differ diff --git a/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-topology.png b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-topology.png new file mode 100644 index 0000000..faeea95 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-topology.png differ diff --git a/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-transport.png b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-transport.png new file mode 100644 index 0000000..510a72b Binary files /dev/null and b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts-transport.png differ diff --git a/hack/k8s-cw-staging-cmri/x-lite-preference-accounts.png b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts.png new file mode 100644 index 0000000..7c850e7 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/x-lite-preference-accounts.png differ diff --git a/hack/k8s-cw-staging-cmri/zoiper-contact.png b/hack/k8s-cw-staging-cmri/zoiper-contact.png new file mode 100644 index 0000000..e5345d1 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/zoiper-contact.png differ diff --git a/hack/k8s-cw-staging-cmri/zoiper-dialup.png b/hack/k8s-cw-staging-cmri/zoiper-dialup.png new file mode 100644 index 0000000..a4581fd Binary files /dev/null and b/hack/k8s-cw-staging-cmri/zoiper-dialup.png differ diff --git a/hack/k8s-cw-staging-cmri/zoiper-preference-accounts-advanced.png b/hack/k8s-cw-staging-cmri/zoiper-preference-accounts-advanced.png new file mode 100644 index 0000000..7cf002f Binary files /dev/null and b/hack/k8s-cw-staging-cmri/zoiper-preference-accounts-advanced.png differ diff --git a/hack/k8s-cw-staging-cmri/zoiper-preference-accounts-general.png b/hack/k8s-cw-staging-cmri/zoiper-preference-accounts-general.png new file mode 100644 index 0000000..ef3a795 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/zoiper-preference-accounts-general.png differ diff --git a/hack/k8s-cw-staging-cmri/zoiper-preference-advanced-network.png b/hack/k8s-cw-staging-cmri/zoiper-preference-advanced-network.png new file mode 100644 index 0000000..acd8fd2 Binary files /dev/null and b/hack/k8s-cw-staging-cmri/zoiper-preference-advanced-network.png differ diff --git a/hack/kubernetes-clearwater/centos-devops.md b/hack/kubernetes-clearwater/centos-devops.md new file mode 100755 index 0000000..c79b6f9 --- /dev/null +++ b/hack/kubernetes-clearwater/centos-devops.md @@ -0,0 +1,179 @@ +About +====== + +Here is working around CentOS development tools, such as *gcc, make,* and so on + +ISO media +------------- + +Using packages from ISO media + + [vagrant@localhost ~]$ cat /etc/yum.repos.d/CentOS-Media.repo + # CentOS-Media.repo + # + # This repo can be used with mounted DVD media, verify the mount point for + # CentOS-7. You can use this repo and yum to install items directly off the + # DVD ISO that we release. + # + # To use this repo, put in your DVD and use it with the other repos too: + # yum --enablerepo=c7-media [command] + # + # or for ONLY the media repo, do this: + # + # yum --disablerepo=\* --enablerepo=c7-media [command] + + [c7-media] + name=CentOS-$releasever - Media + baseurl=file:///media/CentOS/ + file:///media/cdrom/ + file:///media/cdrecorder/ + gpgcheck=1 + enabled=0 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + + [vagrant@localhost ~]$ sudo mkdir -p /media/CentOS + + [vagrant@localhost ~]$ sudo mount -t iso9660 -o loop /vagrant/CentOS-7-x86_64-Everything-1511.iso /media/CentOS + mount: /dev/loop2 is write-protected, mounting read-only + + [vagrant@localhost ~]$ ls /media/CentOS/ + CentOS_BuildTag GPL LiveOS RPM-GPG-KEY-CentOS-7 + EFI images Packages RPM-GPG-KEY-CentOS-Testing-7 + EULA isolinux repodata TRANS.TBL + + [vagrant@localhost ~]$ sudo yum --disablerepo=\* --enablerepo=c7-media repolist + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * c7-media: + c7-media/primary_db | 2.8 MB 00:00 + repo id repo name status + c7-media CentOS-7 - Media 0 + repolist: 0 + +* Tools + +For *net-tools* + + [vagrant@localhost ~]$ sudo yum list net-tools + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * extras: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + Available Packages + net-tools.x86_64 2.0-0.17.20131004git.el7 base + + [vagrant@localhost ~]$ sudo mount /dev/sr0 /media/CentOS + mount: /dev/sr0 is write-protected, mounting read-only + + [vagrant@localhost ~]$ sudo yum --disablerepo=\* --enablerepo=c7-media install net-tools + Loaded plugins: fastestmirror + c7-media | 3.6 kB 00:00 + Not using downloaded repomd.xml because it is older than what we have: + Current : Wed Dec 9 23:14:09 2015 + Downloaded: Wed Dec 9 22:35:45 2015 + Loading mirror speeds from cached hostfile + * c7-media: + Resolving Dependencies + --> Running transaction check + ---> Package net-tools.x86_64 0:2.0-0.17.20131004git.el7 will be installed + --> Finished Dependency Resolution + + Dependencies Resolved + + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Installing: + net-tools x86_64 2.0-0.17.20131004git.el7 c7-media 304 k + + Transaction Summary + ================================================================================ + Install 1 Package + + [vagrant@localhost ~]$ sudo netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 127.0.0.1:2380 0.0.0.0:* LISTEN 998/etcd + tcp 0 0 127.0.0.1:8080 0.0.0.0:* LISTEN 1225/kube-apiserver + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 993/sshd + tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 1430/master + tcp 0 0 127.0.0.1:7001 0.0.0.0:* LISTEN 998/etcd + tcp6 0 0 :::6443 :::* LISTEN 1225/kube-apiserver + tcp6 0 0 :::2379 :::* LISTEN 998/etcd + tcp6 0 0 :::22 :::* LISTEN 993/sshd + tcp6 0 0 ::1:25 :::* LISTEN 1430/master + +For *wget* + + [vagrant@localhost ~]$ sudo yum --disablerepo=\* --enablerepo=c7-media install wget + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * c7-media: + Resolving Dependencies + --> Running transaction check + ---> Package wget.x86_64 0:1.14-10.el7_0.1 will be installed + --> Finished Dependency Resolution + + Dependencies Resolved + + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Installing: + wget x86_64 1.14-10.el7_0.1 c7-media 545 k + + Transaction Summary + ================================================================================ + Install 1 Package + + [vagrant@localhost ~]$ wget + wget: missing URL + Usage: wget [OPTION]... [URL]... + + Try `wget --help' for more options. + +For __NTP__ + + [tangfx@localhost k8s-manifests]$ sudo dnf install chrony + Last metadata expiration check: 2:34:08 ago on Fri Dec 9 04:46:01 2016. + Dependencies resolved. + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Installing: + chrony x86_64 2.4.1-1.fc23 updates 225 k + timedatex x86_64 0.3-3.fc23 fedora 30 k + + Transaction Summary + ================================================================================ + Install 2 Packages + + [tangfx@localhost k8s-manifests]$ sudo systemctl enable chronyd.service + [tangfx@localhost k8s-manifests]$ sudo systemctl start chronyd.service + [tangfx@localhost k8s-manifests]$ sudo timedatectl + Local time: Fri 2016-12-09 07:21:32 CST + Universal time: Thu 2016-12-08 23:21:32 UTC + RTC time: Thu 2016-12-08 23:21:32 + Time zone: Asia/Shanghai (CST, +0800) + Network time on: yes + NTP synchronized: no + RTC in local TZ: no + + [tangfx@localhost k8s-manifests]$ sudo timedatectl set-timezone UTC + + [tangfx@localhost k8s-manifests]$ sudo chronyc -a makestep + 200 OK + + [tangfx@localhost k8s-manifests]$ sudo timedatectl set-ntp yes + + [tangfx@localhost k8s-manifests]$ sudo date --set "2016-12-09 09:08:37" --utc Fri Dec 9 09:08:37 UTC 2016 + [tangfx@localhost k8s-manifests]$ sudo hwclock --set --utc --date "`date`" + [tangfx@localhost k8s-manifests]$ sudo timedatectl + Local time: Fri 2016-12-09 09:09:12 UTC + Universal time: Fri 2016-12-09 09:09:12 UTC + RTC time: Fri 2016-12-09 09:09:12 + Time zone: UTC (UTC, +0000) + Network time on: yes + NTP synchronized: yes + RTC in local TZ: no diff --git a/hack/kubernetes-clearwater/clearwater/README.md b/hack/kubernetes-clearwater/clearwater/README.md new file mode 100755 index 0000000..0594653 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/README.md @@ -0,0 +1,544 @@ +# Content + +## Build + + [tangfx@localhost alt-ops]$ make + +## Deploy + +Etcd + + [tangfx@localhost k8s-manifests]$ ~/bin/kubectl create -f etcd-service.yaml + service "etcd" created + + [tangfx@localhost k8s-manifests]$ ~/bin/kubectl create -f etcd-deployment.yaml + deployment "etcd" created + +Memcache, Chronos & Cassandra + + [tangfx@localhost k8s-manifests]$ ~/bin/kubectl create -f memcached-deployment.yaml -f chronos-deployment.yaml -f cassandra-deployment.yaml + deployment "memcached" created + deployment "chronos" created + deployment "cassandra" created + +* Validate + +For example _cassandra_ + + root@cassandra-4015560737-f3241:~# etcdctl --endpoint=http://localhost:4000 ls /clearwater/ + /clearwater/site1 + /clearwater/node_type_cassandra + root@cassandra-4015560737-f3241:~# etcdctl --endpoint=http://localhost:4000 get /clearwater/node_type_cassandra/clustering/cassandra + {"10.120.8.4": "joining, acknowledged change"} + + root@cassandra-4015560737-f3241:~# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 0.0.0.0:53 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 7/sshd + tcp6 0 0 :::4000 :::* LISTEN - + tcp6 0 0 :::9160 :::* LISTEN 1031/java + tcp6 0 0 :::9042 :::* LISTEN 1031/java + tcp6 0 0 :::53 :::* LISTEN - + tcp6 0 0 127.0.0.1:43414 :::* LISTEN 1031/java + tcp6 0 0 :::22 :::* LISTEN 7/sshd + tcp6 0 0 10.120.8.4:7000 :::* LISTEN 1031/java + tcp6 0 0 127.0.0.1:7199 :::* LISTEN 1031/java + + root@cassandra-4015560737-f3241:~# tail /var/log/cassandra/system.log + INFO [CompactionExecutor:1] 2016-12-12 19:08:25,648 CompactionTask.java:141 - Compacting [SSTableReader(path='/var/lib/cassandra/data/system/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697/system-schema_columnfamilies-ka-24-Data.db'), SSTableReader(path='/var/lib/cassandra/data/system/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697/system-schema_columnfamilies-ka-22-Data.db'), SSTableReader(path='/var/lib/cassandra/data/system/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697/system-schema_columnfamilies-ka-21-Data.db'), SSTableReader(path='/var/lib/cassandra/data/system/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697/system-schema_columnfamilies-ka-23-Data.db')] + INFO [CompactionExecutor:2] 2016-12-12 19:08:25,732 CompactionTask.java:274 - Compacted 4 sstables to [/var/lib/cassandra/data/system/schema_keyspaces-b0f2235744583cdb9631c43e59ce3676/system-schema_keyspaces-ka-29,]. 848 bytes to 339 (~39% of original) in 76ms = 0.004254MB/s. 8 total partitions merged to 5. Partition merge counts were {1:3, 2:1, 3:1, } + INFO [SharedPool-Worker-5] 2016-12-12 19:08:25,740 MigrationManager.java:295 - Update ColumnFamily 'homestead_cache/impi' From org.apache.cassandra.config.CFMetaData@59def0df[cfId=584bd7b0-c09e-11e6-85b4-c788056216fa,ksName=homestead_cache,cfName=impi,cfType=Standard,comparator=org.apache.cassandra.db.marshal.UTF8Type,comment=,readRepairChance=1.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=digest_realm, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=digest_ha1, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=digest_qop, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=private_id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}],compactionStrategyClass=class org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.01,memtableFlushPeriod=0,caching={"keys":"ALL", "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=50.0ms,droppedColumns={},triggers=[],isDense=false] To org.apache.cassandra.config.CFMetaData@3a213a8d[cfId=584bd7b0-c09e-11e6-85b4-c788056216fa,ksName=homestead_cache,cfName=impi,cfType=Standard,comparator=org.apache.cassandra.db.marshal.UTF8Type,comment=,readRepairChance=1.0,dcLocalReadRepairChance=0.1,gcGraceSeconds=864000,defaultValidator=org.apache.cassandra.db.marshal.BytesType,keyValidator=org.apache.cassandra.db.marshal.UTF8Type,minCompactionThreshold=4,maxCompactionThreshold=32,columnMetadata=[ColumnDefinition{name=digest_realm, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=digest_ha1, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=digest_qop, type=org.apache.cassandra.db.marshal.UTF8Type, kind=REGULAR, componentIndex=null, indexName=null, indexType=null}, ColumnDefinition{name=private_id, type=org.apache.cassandra.db.marshal.UTF8Type, kind=PARTITION_KEY, componentIndex=null, indexName=null, indexType=null}],compactionStrategyClass=class org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy,compactionStrategyOptions={},compressionParameters={sstable_compression=org.apache.cassandra.io.compress.LZ4Compressor},bloomFilterFpChance=0.01,memtableFlushPeriod=0,caching={"keys":"ALL", "rows_per_partition":"NONE"},defaultTimeToLive=0,minIndexInterval=128,maxIndexInterval=2048,speculativeRetry=50.0ms,droppedColumns={},triggers=[],isDense=false] + INFO [MigrationStage:1] 2016-12-12 19:08:25,744 ColumnFamilyStore.java:905 - Enqueuing flush of schema_keyspaces: 489 (0%) on-heap, 0 (0%) off-heap + INFO [MemtableFlushWriter:2] 2016-12-12 19:08:25,744 Memtable.java:347 - Writing Memtable-schema_keyspaces@3715952(0.143KiB serialized bytes, 3 ops, 0%/0% of on/off-heap limit) + INFO [MemtableFlushWriter:2] 2016-12-12 19:08:25,745 Memtable.java:382 - Completed flushing /var/lib/cassandra/data/system/schema_keyspaces-b0f2235744583cdb9631c43e59ce3676/system-schema_keyspaces-tmp-ka-30-Data.db (0.000KiB) for commitlog position ReplayPosition(segmentId=1481569683573, position=179867) + INFO [MigrationStage:1] 2016-12-12 19:08:25,757 ColumnFamilyStore.java:905 - Enqueuing flush of schema_columnfamilies: 5759 (0%) on-heap, 0 (0%) off-heap + INFO [MemtableFlushWriter:1] 2016-12-12 19:08:25,757 Memtable.java:347 - Writing Memtable-schema_columnfamilies@729751315(1.136KiB serialized bytes, 27 ops, 0%/0% of on/off-heap limit) + INFO [MemtableFlushWriter:1] 2016-12-12 19:08:25,758 Memtable.java:382 - Completed flushing /var/lib/cassandra/data/system/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697/system-schema_columnfamilies-tmp-ka-26-Data.db (0.000KiB) for commitlog position ReplayPosition(segmentId=1481569683573, position=179867) + INFO [CompactionExecutor:1] 2016-12-12 19:08:25,786 CompactionTask.java:274 - Compacted 4 sstables to [/var/lib/cassandra/data/system/schema_columnfamilies-45f5b36024bc3f83a3631034ea4fa697/system-schema_columnfamilies-ka-25,]. 12,772 bytes to 10,232 (~80% of original) in 134ms = 0.072821MB/s. 8 total partitions merged to 5. Partition merge counts were {1:3, 2:1, 3:1, } + + +First + + [tangfx@localhost k8s-manifests]$ ~/bin/kubectl create -f homestead-deployment.yaml -f homer-deployment.yaml -f ralf-deployment.yaml -f sprout-deployment.yaml + deployment "homestead" created + deployment "homer" created + deployment "ralf" created + deployment "sprout" created + +_homestead_ + + root@homestead-262042966-2o0pu:~# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 0.0.0.0:53 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 8/sshd + tcp 0 0 127.0.0.1:8888 0.0.0.0:* LISTEN - + tcp 0 0 10.120.8.6:8888 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:5658 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:3868 0.0.0.0:* LISTEN - + tcp6 0 0 :::4000 :::* LISTEN - + tcp6 0 0 :::80 :::* LISTEN 225/nginx -g daemon + tcp6 0 0 :::53 :::* LISTEN - + tcp6 0 0 :::22 :::* LISTEN 8/sshd + tcp6 0 0 :::8889 :::* LISTEN 225/nginx -g daemon + tcp6 0 0 :::5658 :::* LISTEN - + tcp6 0 0 :::3868 :::* LISTEN - + + root@homestead-262042966-2o0pu:~# ls /var/log/homestead/homestead_ + homestead_20161212T190000Z.txt homestead_current.txt + + root@homestead-262042966-2o0pu:~# tail /var/log/homestead/homestead_current.txt 12-12-2016 19:10:47.258 UTC Status handlers.cpp:88: Server-Name: sip:sprout.default.svc.cluster.local:5054;transport=tcp + 12-12-2016 19:10:47.258 UTC Status httpstack.cpp:58: Constructing HTTP stack with 50 threads + 12-12-2016 19:10:47.258 UTC Status httpstack.cpp:164: Binding HTTP TCP socket: address=10.120.8.6, port=8888 + 12-12-2016 19:10:47.533 UTC Status httpstack.cpp:58: Constructing HTTP stack with 5 threads + 12-12-2016 19:10:47.533 UTC Status httpstack.cpp:235: Binding HTTP unix socket: path=/tmp/homestead-http-mgmt-socket + 12-12-2016 19:10:47.562 UTC Status main.cpp:941: Start-up complete - wait for termination signal + 12-12-2016 19:11:01.404 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:11:16.425 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:11:31.431 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:11:46.451 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + +_homer_ + + root@homer-2080525652-x5vvy:~# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 0.0.0.0:53 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 8/sshd + tcp6 0 0 :::4000 :::* LISTEN - + tcp6 0 0 :::80 :::* LISTEN 153/nginx -g daemon + tcp6 0 0 :::7888 :::* LISTEN 153/nginx -g daemon + tcp6 0 0 :::53 :::* LISTEN - + tcp6 0 0 :::22 :::* LISTEN 8/sshd + + root@homer-2080525652-x5vvy:~# cat /var/log/homer/homer_20161212T190000Z.txt + 12-12-2016 19:10:27.513 UTC INFO utils.py:441: Acquired exclusive lock on /var/run/homer.pid.lockfile + 12-12-2016 19:10:27.514 UTC INFO main.py:75: Going to listen for HTTP on UNIX socket /tmp/.homer-sock-0 + + +_ralf_ + + root@ralf-3065138204-q1djm:~# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 127.0.0.1:10888 0.0.0.0:* LISTEN - + tcp 0 0 10.120.8.7:10888 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:53 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 7/sshd + tcp 0 0 0.0.0.0:5659 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:3869 0.0.0.0:* LISTEN - + tcp6 0 0 :::4000 :::* LISTEN - + tcp6 0 0 :::53 :::* LISTEN - + tcp6 0 0 :::22 :::* LISTEN 7/sshd + tcp6 0 0 :::5659 :::* LISTEN - + tcp6 0 0 :::3869 :::* LISTEN - + + root@ralf-3065138204-q1djm:~# tail -50 /var/log/ralf/ralf_current.txt + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Default trace level .... : +3 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Configuration file ..... : /var/lib/ralf/ralf.conf + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Diameter Identity ...... : 10.120.8.7 (l:10) + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Diameter Realm ......... : ralf.default.svc.cluster.local (l:30) + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Tc Timer ............... : 30 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Tw Timer ............... : 30 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Local port ............. : 3869 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Local secure port ...... : 5659 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Number of SCTP streams . : 30 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Number of clients thr .. : 5 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Number of app threads .. : 4 + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Local endpoints ........ : Default (use all available) + 12-12-2016 19:10:27.651 UTC Status freeDiameter: Local applications ..... : (none) + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Flags : - IP ........... : Enabled + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - IPv6 ......... : Enabled + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - Relay app .... : Enabled + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - TCP .......... : Enabled + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - SCTP ......... : DISABLED + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - Pref. proto .. : SCTP + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - TLS method ... : Separate port + 12-12-2016 19:10:27.652 UTC Status freeDiameter: TLS : - Certificate .. : /var/lib/ralf/cert.pem + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - Private key .. : /var/lib/ralf/privkey.pem + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - CA (trust) ... : /var/lib/ralf/ca.pem (1 certs) + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - CRL .......... : (none) + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - Priority ..... : (default: 'NORMAL') + 12-12-2016 19:10:27.652 UTC Status freeDiameter: - DH bits ...... : 1024 + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Origin-State-Id ........ : 1481569827 + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Loaded extensions: '/usr/share/clearwater/ralf/lib/freeDiameter//dbg_monitor.fdx'[(no config file)], loaded + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Loaded extensions: '/usr/share/clearwater/ralf/lib/freeDiameter//dict_nasreq.fdx'[(no config file)], loaded + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Loaded extensions: '/usr/share/clearwater/ralf/lib/freeDiameter//dict_sip.fdx'[(no config file)], loaded + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Loaded extensions: '/usr/share/clearwater/ralf/lib/freeDiameter//dict_dcca.fdx'[(no config file)], loaded + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Loaded extensions: '/usr/share/clearwater/ralf/lib/freeDiameter//dict_dcca_3gpp.fdx'[(no config file)], loaded + 12-12-2016 19:10:27.652 UTC Status freeDiameter: {signal:12}'dbg_monitor'->0x7fa33040fe70 + 12-12-2016 19:10:27.652 UTC Status diameterstack.cpp:573: Starting Diameter stack + 12-12-2016 19:10:27.652 UTC Status freeDiameter: Local server address(es): 10.120.8.7{---L-} + 12-12-2016 19:10:27.652 UTC Status dnscachedresolver.cpp:150: Creating Cached Resolver using servers: + 12-12-2016 19:10:27.652 UTC Status dnscachedresolver.cpp:160: 10.123.240.10 + 12-12-2016 19:10:27.652 UTC Status main.cpp:702: Creating connection to Chronos at chronos.default.svc.cluster.local:7253 using ralf.default.svc.cluster.local:10888 as the callback URI + 12-12-2016 19:10:27.652 UTC Status a_record_resolver.cpp:54: Created ARecordResolver + 12-12-2016 19:10:27.652 UTC Status http_connection_pool.cpp:50: Connection pool will use a response timeout of 500ms + 12-12-2016 19:10:27.653 UTC Status httpconnection.h:71: Configuring HTTP Connection + 12-12-2016 19:10:27.653 UTC Status httpconnection.h:72: Connection created for server chronos.default.svc.cluster.local:7253 + 12-12-2016 19:10:27.653 UTC Status httpstack.cpp:58: Constructing HTTP stack with 50 threads + 12-12-2016 19:10:27.653 UTC Status httpstack.cpp:164: Binding HTTP TCP socket: address=10.120.8.7, port=10888 + 12-12-2016 19:10:27.933 UTC Status diameterresolver.cpp:62: Created Diameter resolver + 12-12-2016 19:10:27.938 UTC Error dnscachedresolver.cpp:846: Failed to retrieve record for default.svc.cluster.local: Domain name not found + 12-12-2016 19:10:27.941 UTC Error dnscachedresolver.cpp:846: Failed to retrieve record for _diameter._tcp.default.svc.cluster.local: Domain name not found + 12-12-2016 19:10:27.941 UTC Error dnscachedresolver.cpp:846: Failed to retrieve record for _diameter._sctp.default.svc.cluster.local: Domain name not found + 12-12-2016 19:10:27.941 UTC Error diameterstack.cpp:854: No Diameter peers have been found + 12-12-2016 19:15:27.941 UTC Error diameterstack.cpp:854: No Diameter peers have been found + +_sprout_ + + root@sprout-2743169544-djjbf:~# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 10.120.94.3:5052 0.0.0.0:* LISTEN - + tcp 0 0 10.120.94.3:5053 0.0.0.0:* LISTEN - + tcp 0 0 10.120.94.3:5054 0.0.0.0:* LISTEN - + tcp 0 0 10.120.94.3:5055 0.0.0.0:* LISTEN - + tcp 0 0 127.0.0.1:9888 0.0.0.0:* LISTEN - + tcp 0 0 10.120.94.3:9888 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:53 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 8/sshd + tcp6 0 0 :::4000 :::* LISTEN - + tcp6 0 0 :::53 :::* LISTEN - + tcp6 0 0 :::22 :::* LISTEN 8/sshd + + root@sprout-2743169544-djjbf:~# cat /var/log/sprout/sprout_current.txt + 12-12-2016 19:10:27.619 UTC Status utils.cpp:620: Log level set to 2 + 12-12-2016 19:10:27.619 UTC Status main.cpp:1411: Access logging enabled to /var/log/sprout + 12-12-2016 19:10:27.620 UTC Warning main.cpp:1456: SAS server option was invalid or not configured - SAS is disabled + 12-12-2016 19:10:27.621 UTC Status snmp_agent.cpp:117: AgentX agent initialised + 12-12-2016 19:10:28.685 UTC Status utils.cpp:620: Log level set to 2 + 12-12-2016 19:10:28.685 UTC Status main.cpp:1411: Access logging enabled to /var/log/sprout + 12-12-2016 19:10:28.686 UTC Warning main.cpp:1456: SAS server option was invalid or not configured - SAS is disabled + 12-12-2016 19:10:28.686 UTC Status snmp_agent.cpp:117: AgentX agent initialised + 12-12-2016 19:10:28.688 UTC Status load_monitor.cpp:105: Constructing LoadMonitor + 12-12-2016 19:10:28.688 UTC Status load_monitor.cpp:106: Target latency (usecs) : 100000 + 12-12-2016 19:10:28.688 UTC Status load_monitor.cpp:107: Max bucket size : 1000 + 12-12-2016 19:10:28.688 UTC Status load_monitor.cpp:108: Initial token fill rate/s: 250.000000 + 12-12-2016 19:10:28.688 UTC Status load_monitor.cpp:109: Min token fill rate/s : 10.000000 + 12-12-2016 19:10:28.688 UTC Status dnscachedresolver.cpp:150: Creating Cached Resolver using servers: + 12-12-2016 19:10:28.688 UTC Status dnscachedresolver.cpp:160: 10.123.240.10 + 12-12-2016 19:10:28.688 UTC Status sipresolver.cpp:60: Created SIP resolver + 12-12-2016 19:10:28.710 UTC Status stack.cpp:419: Listening on port 5052 + 12-12-2016 19:10:28.710 UTC Status stack.cpp:419: Listening on port 5053 + 12-12-2016 19:10:28.710 UTC Status stack.cpp:419: Listening on port 5054 + 12-12-2016 19:10:28.711 UTC Status stack.cpp:419: Listening on port 5055 + 12-12-2016 19:10:28.711 UTC Status stack.cpp:852: Local host aliases: + 12-12-2016 19:10:28.711 UTC Status stack.cpp:859: 10.120.94.3 + 12-12-2016 19:10:28.711 UTC Status stack.cpp:859: sprout.default.svc.cluster.local + 12-12-2016 19:10:28.711 UTC Status stack.cpp:859: 10.120.94.3 + 12-12-2016 19:10:28.711 UTC Status stack.cpp:859: bgcf.sprout.default.svc.cluster.local + 12-12-2016 19:10:28.711 UTC Status stack.cpp:859: mmtel.sprout.default.svc.cluster.local + 12-12-2016 19:10:28.711 UTC Status stack.cpp:859: 10.120.94.3 + 12-12-2016 19:10:28.711 UTC Status stack.cpp:859: sprout.default.svc.cluster.local + 12-12-2016 19:10:28.711 UTC Status a_record_resolver.cpp:54: Created ARecordResolver + 12-12-2016 19:10:28.711 UTC Status http_connection_pool.cpp:50: Connection pool will use a response timeout of 500ms + 12-12-2016 19:10:28.714 UTC Status httpconnection.h:71: Configuring HTTP Connection + 12-12-2016 19:10:28.714 UTC Status httpconnection.h:72: Connection created for server ralf.default.svc.cluster.local:10888 + 12-12-2016 19:10:28.714 UTC Status main.cpp:1808: Creating connection to HSS homestead.default.svc.cluster.local:8888 + 12-12-2016 19:10:28.714 UTC Status http_connection_pool.cpp:50: Connection pool will use a response timeout of 500ms + 12-12-2016 19:10:28.714 UTC Status httpconnection.h:71: Configuring HTTP Connection + 12-12-2016 19:10:28.714 UTC Status httpconnection.h:72: Connection created for server homestead.default.svc.cluster.local:8888 + 12-12-2016 19:10:28.714 UTC Status main.cpp:1839: Setting up ENUM service to do default TEL->SIP URI translation + 12-12-2016 19:10:28.714 UTC Status main.cpp:1922: Creating connection to Chronos chronos.default.svc.cluster.local:7253 using sprout.default.svc.cluster.local:9888 as the callback URI + 12-12-2016 19:10:28.714 UTC Status http_connection_pool.cpp:50: Connection pool will use a response timeout of 500ms + 12-12-2016 19:10:28.714 UTC Status httpconnection.h:71: Configuring HTTP Connection + 12-12-2016 19:10:28.714 UTC Status httpconnection.h:72: Connection created for server chronos.default.svc.cluster.local:7253 + 12-12-2016 19:10:28.714 UTC Status main.cpp:1935: Using memcached compatible store with ASCII protocol + 12-12-2016 19:10:28.714 UTC Status memcached_config.cpp:70: Reloading memcached configuration from '/etc/clearwater/cluster_settings' + 12-12-2016 19:10:28.714 UTC Status memcached_config.cpp:109: servers=10.120.8.2:11211 + 12-12-2016 19:10:28.714 UTC Status memcachedstore.cpp:430: Updating memcached store configuration + 12-12-2016 19:10:28.715 UTC Status memcachedstore.cpp:451: Finished preparing new view, so flag that workers should switch to it + 12-12-2016 19:10:28.715 UTC Status main.cpp:1952: Using remote memcached compatible store with ASCII protocol + 12-12-2016 19:10:28.715 UTC Status memcached_config.cpp:70: Reloading memcached configuration from '/etc/clearwater/remote_cluster_settings' + 12-12-2016 19:10:28.715 UTC Status memcached_config.cpp:109: servers= + 12-12-2016 19:10:28.715 UTC Status memcachedstore.cpp:430: Updating memcached store configuration + 12-12-2016 19:10:28.715 UTC Status memcachedstore.cpp:451: Finished preparing new view, so flag that workers should switch to it + 12-12-2016 19:10:28.715 UTC Warning main.cpp:1962: Remote cluster settings file '/etc/clearwater/remote_cluster_settings' does not contain a valid set of servers + 12-12-2016 19:10:28.715 UTC Status httpstack.cpp:58: Constructing HTTP stack with 50 threads + 12-12-2016 19:10:28.715 UTC Status httpstack.cpp:58: Constructing HTTP stack with 5 threads + 12-12-2016 19:10:28.715 UTC Status main.cpp:2049: Initialise S-CSCF authentication module + 12-12-2016 19:10:28.715 UTC Status pluginloader.cpp:63: Loading plug-ins from /usr/share/clearwater/sprout/plugins + 12-12-2016 19:10:28.715 UTC Status pluginloader.cpp:93: Attempt to load plug-in /usr/share/clearwater/sprout/plugins/sprout_bgcf.so + 12-12-2016 19:10:28.715 UTC Status bgcfplugin.cpp:101: BGCF plugin enabled + 12-12-2016 19:10:28.715 UTC Status bgcfservice.cpp:68: No BGCF configuration (file ./bgcf.json does not exist) + 12-12-2016 19:10:28.715 UTC Status pluginloader.cpp:105: Plug-in /usr/share/clearwater/sprout/plugins/sprout_bgcf.so has loaded 1 Sproutlets + 12-12-2016 19:10:28.715 UTC Status pluginloader.cpp:128: Loaded sproutlet bgcf using API version 1 + 12-12-2016 19:10:28.715 UTC Status pluginloader.cpp:93: Attempt to load plug-in /usr/share/clearwater/sprout/plugins/sprout_icscf.so + 12-12-2016 19:10:28.716 UTC Status icscfplugin.cpp:97: I-CSCF plugin enabled + 12-12-2016 19:10:28.716 UTC Status scscfselector.cpp:72: No S-CSCF configuration data (file ./s-cscf.json does not exist) + 12-12-2016 19:10:28.716 UTC Warning scscfselector.cpp:166: The S-CSCF json file is empty/invalid. Using default values + 12-12-2016 19:10:28.716 UTC Status pluginloader.cpp:105: Plug-in /usr/share/clearwater/sprout/plugins/sprout_icscf.so has loaded 1 Sproutlets + 12-12-2016 19:10:28.716 UTC Status pluginloader.cpp:128: Loaded sproutlet icscf using API version 1 + 12-12-2016 19:10:28.716 UTC Status pluginloader.cpp:93: Attempt to load plug-in /usr/share/clearwater/sprout/plugins/sprout_mmtel_as.so + 12-12-2016 19:10:28.717 UTC Status mmtelasplugin.cpp:89: MMTel AS plugin enabled + 12-12-2016 19:10:28.717 UTC Status mmtelasplugin.cpp:98: Creating connection to XDMS homer.default.svc.cluster.local:7888 + 12-12-2016 19:10:28.717 UTC Status http_connection_pool.cpp:50: Connection pool will use a response timeout of 500ms + 12-12-2016 19:10:28.717 UTC Status httpconnection.h:71: Configuring HTTP Connection + 12-12-2016 19:10:28.717 UTC Status httpconnection.h:72: Connection created for server homer.default.svc.cluster.local:7888 + 12-12-2016 19:10:28.717 UTC Status pluginloader.cpp:105: Plug-in /usr/share/clearwater/sprout/plugins/sprout_mmtel_as.so has loaded 1 Sproutlets + 12-12-2016 19:10:28.717 UTC Status pluginloader.cpp:128: Loaded sproutlet mmtel using API version 1 + 12-12-2016 19:10:28.717 UTC Status pluginloader.cpp:93: Attempt to load plug-in /usr/share/clearwater/sprout/plugins/sprout_scscf.so + 12-12-2016 19:10:28.718 UTC Status scscfplugin.cpp:100: S-CSCF plugin enabled + 12-12-2016 19:10:28.718 UTC Status pluginloader.cpp:105: Plug-in /usr/share/clearwater/sprout/plugins/sprout_scscf.so has loaded 1 Sproutlets + 12-12-2016 19:10:28.718 UTC Status pluginloader.cpp:128: Loaded sproutlet scscf using API version 1 + 12-12-2016 19:10:28.718 UTC Status pluginloader.cpp:158: Finished loading plug-ins + 12-12-2016 19:10:28.719 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:10:28.720 UTC Status stack.cpp:173: PJSIP thread started + 12-12-2016 19:10:29.002 UTC Status httpstack.cpp:164: Binding HTTP TCP socket: address=10.120.94.3, port=9888 + 12-12-2016 19:10:29.030 UTC Status httpstack.cpp:235: Binding HTTP unix socket: path=/tmp/sprout-http-mgmt-socket + 12-12-2016 19:10:43.730 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:10:58.741 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:11:13.754 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:11:28.766 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:11:43.781 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:11:58.796 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + +Then + + [tangfx@localhost k8s-manifests]$ ~/bin/kubectl create -f bono-deployment-dockerproxy.yaml -f ellis-deployment-dockerproxy.yaml + deployment "bono" created + deployment "ellis" created + +_bono_ + + root@bono-2501630498-nntnh:~# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 10.120.8.8:5058 0.0.0.0:* LISTEN - + tcp 0 0 10.120.8.8:5060 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:5062 0.0.0.0:* LISTEN - + tcp 0 0 127.0.0.1:8080 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:53 0.0.0.0:* LISTEN - + tcp 0 0 10.120.8.8:3478 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 7/sshd + tcp6 0 0 :::4000 :::* LISTEN - + tcp6 0 0 :::53 :::* LISTEN - + tcp6 0 0 :::22 :::* LISTEN 7/sshd + + root@bono-2501630498-nntnh:~# cat /var/log/bono/bono_current.txt + 12-12-2016 19:21:27.203 UTC Status utils.cpp:620: Log level set to 2 + 12-12-2016 19:21:27.203 UTC Status main.cpp:1411: Access logging enabled to /var/log/bono + 12-12-2016 19:21:27.204 UTC Warning main.cpp:1456: SAS server option was invalid or not configured - SAS is disabled + 12-12-2016 19:21:27.204 UTC Warning main.cpp:1532: A registration expiry period should not be specified for P-CSCF + 12-12-2016 19:21:27.204 UTC Status snmp_agent.cpp:117: AgentX agent initialised + 12-12-2016 19:21:27.206 UTC Status load_monitor.cpp:105: Constructing LoadMonitor + 12-12-2016 19:21:27.206 UTC Status load_monitor.cpp:106: Target latency (usecs) : 100000 + 12-12-2016 19:21:27.206 UTC Status load_monitor.cpp:107: Max bucket size : 1000 + 12-12-2016 19:21:27.206 UTC Status load_monitor.cpp:108: Initial token fill rate/s: 100.000000 + 12-12-2016 19:21:27.206 UTC Status load_monitor.cpp:109: Min token fill rate/s : 10.000000 + 12-12-2016 19:21:27.206 UTC Status dnscachedresolver.cpp:150: Creating Cached Resolver using servers: + 12-12-2016 19:21:27.206 UTC Status dnscachedresolver.cpp:160: 10.123.240.10 + 12-12-2016 19:21:27.206 UTC Status sipresolver.cpp:60: Created SIP resolver + 12-12-2016 19:21:27.223 UTC Status stack.cpp:419: Listening on port 5058 + 12-12-2016 19:21:27.223 UTC Status stack.cpp:419: Listening on port 5060 + 12-12-2016 19:21:27.224 UTC Status stack.cpp:852: Local host aliases: + 12-12-2016 19:21:27.224 UTC Status stack.cpp:859: 10.120.8.8 + 12-12-2016 19:21:27.224 UTC Status stack.cpp:859: 10.120.8.8 + 12-12-2016 19:21:27.224 UTC Status stack.cpp:859: 10.120.8.8 + 12-12-2016 19:21:27.224 UTC Status stack.cpp:859: + 12-12-2016 19:21:27.224 UTC Status a_record_resolver.cpp:54: Created ARecordResolver + 12-12-2016 19:21:27.224 UTC Status http_connection_pool.cpp:50: Connection pool will use a response timeout of 500ms + 12-12-2016 19:21:27.228 UTC Status httpconnection.h:71: Configuring HTTP Connection + 12-12-2016 19:21:27.228 UTC Status httpconnection.h:72: Connection created for server ralf.default.svc.cluster.local:10888 + 12-12-2016 19:21:27.239 UTC Status sip_connection_pool.cpp:72: Creating connection pool to sprout.default.svc.cluster.local:5052 + 12-12-2016 19:21:27.239 UTC Status sip_connection_pool.cpp:73: connections = 50, recycle time = 600 +/- 120 seconds + 12-12-2016 19:21:27.245 UTC Status bono.cpp:3330: Create list of PBXes + 12-12-2016 19:21:27.245 UTC Status main.cpp:1922: Creating connection to Chronos 127.0.0.1:7253 using 127.0.0.1:9888 as the callback URI + 12-12-2016 19:21:27.245 UTC Status http_connection_pool.cpp:50: Connection pool will use a response timeout of 500ms + 12-12-2016 19:21:27.246 UTC Status httpconnection.h:71: Configuring HTTP Connection + 12-12-2016 19:21:27.246 UTC Status httpconnection.h:72: Connection created for server 127.0.0.1:7253 + 12-12-2016 19:21:27.246 UTC Status main.cpp:1969: Using local store + 12-12-2016 19:21:27.246 UTC Status httpstack.cpp:58: Constructing HTTP stack with 1 threads + 12-12-2016 19:21:27.246 UTC Status httpstack.cpp:58: Constructing HTTP stack with 5 threads + 12-12-2016 19:21:27.246 UTC Status pluginloader.cpp:63: Loading plug-ins from /usr/share/clearwater/sprout/plugins + 12-12-2016 19:21:27.246 UTC Status pluginloader.cpp:158: Finished loading plug-ins + 12-12-2016 19:21:27.252 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:21:27.252 UTC Status stack.cpp:173: PJSIP thread started + 12-12-2016 19:21:42.265 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:21:57.278 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + 12-12-2016 19:22:12.293 UTC Warning (Net-SNMP): Warning: Failed to connect to the agentx master agent ([NIL]): + + +_ellis_ + + root@ellis-3923004842-g7gzg:~# netstat -tpnl + Active Internet connections (only servers) + Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name + tcp 0 0 127.0.0.1:3306 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:53 0.0.0.0:* LISTEN - + tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 9/sshd + tcp6 0 0 :::4000 :::* LISTEN - + tcp6 0 0 :::80 :::* LISTEN 188/nginx -g daemon + tcp6 0 0 :::53 :::* LISTEN - + tcp6 0 0 :::22 :::* LISTEN 9/sshd + + root@ellis-3923004842-g7gzg:~# cat /var/log/ellis/ellis_20161212T190000Z.txt + 12-12-2016 19:21:28.042 UTC INFO main.py:113: Ellis process starting up + 12-12-2016 19:21:28.419 UTC WARNING iostream.py:507: Connect error on fd 9: ECONNREFUSED + 12-12-2016 19:21:28.424 UTC ERROR homestead.py:68: Failed to ping Homestead at http://homestead.default.svc.cluster.local:8889/ping. Have you configured your HOMESTEAD_URL? + + root@ellis-3923004842-g7gzg:~# ping -c3 homestead.default.svc.cluster.local + PING homestead.default.svc.cluster.local.10.64.33.90.xip.io (10.64.33.90) 56(84) bytes of data. + 64 bytes from localhost (10.64.33.90): icmp_seq=1 ttl=64 time=0.018 ms + 64 bytes from localhost (10.64.33.90): icmp_seq=2 ttl=64 time=0.103 ms + 64 bytes from localhost (10.64.33.90): icmp_seq=3 ttl=64 time=0.157 ms + + --- homestead.default.svc.cluster.local.10.64.33.90.xip.io ping statistics --- + 3 packets transmitted, 3 received, 0% packet loss, time 2039ms + rtt min/avg/max/mdev = 0.018/0.092/0.157/0.058 ms + root@ellis-3923004842-g7gzg:~# ping -c3 homestead + PING homestead.default.svc.cluster.local (10.120.8.6) 56(84) bytes of data. + 64 bytes from 10.120.8.6: icmp_seq=1 ttl=60 time=0.393 ms + 64 bytes from 10.120.8.6: icmp_seq=2 ttl=60 time=0.507 ms + 64 bytes from 10.120.8.6: icmp_seq=3 ttl=60 time=0.324 ms + + --- homestead.default.svc.cluster.local ping statistics --- + 3 packets transmitted, 3 received, 0% packet loss, time 2010ms + rtt min/avg/max/mdev = 0.324/0.408/0.507/0.075 ms + +Last + + vagrant@vagrant-ubuntu-trusty-64:~$ kubectl get pods -o wide + NAME READY STATUS RESTARTS AGE IP NODE + bono-2501630498-nntnh 1/1 Running 1 13h 10.120.8.6 10.64.33.224 + cassandra-4015560737-f3241 1/1 Running 1 13h 10.120.8.7 10.64.33.224 + chronos-1502564329-ln3ba 1/1 Running 1 13h 10.120.8.4 10.64.33.224 + ellis-3923004842-g7gzg 1/1 Running 1 13h 10.120.94.4 10.64.33.90 + etcd-1272536175-ocec0 1/1 Running 0 11s 10.120.8.2 10.64.33.224 + homer-2080525652-x5vvy 1/1 Running 1 13h 10.120.94.6 10.64.33.90 + homestead-262042966-2o0pu 1/1 Running 1 13h 10.120.8.3 10.64.33.224 + memcached-333140447-far6m 1/1 Running 1 13h 10.120.8.5 10.64.33.224 + ralf-3065138204-q1djm 1/1 Running 1 13h 10.120.8.8 10.64.33.224 + sprout-2743169544-djjbf 1/1 Running 1 13h 10.120.94.5 10.64.33.90 + + +### Config + +Cluster + + root@cassandra-4015560737-f3241:~# cat /etc/clearwater/shared_config + # Deployment definitions + home_domain=default.svc.cluster.local + sprout_hostname=sprout.default.svc.cluster.local + hs_hostname=homestead.default.svc.cluster.local:8888 + hs_provisioning_hostname=homestead.default.svc.cluster.local:8889 + xdms_hostname=homer.default.svc.cluster.local:7888 + ralf_hostname=ralf.default.svc.cluster.local:10888 + chronos_hostname=chronos.default.svc.cluster.local + cassandra_hostname=cassandra.default.svc.cluster.local + + # Email server configuration + smtp_smarthost=127.0.0.1 + smtp_username=username + smtp_password=password + email_recovery_sender=clearwater@default.svc.cluster.local + + # I-CSCF/S-CSCF configuration + upstream_hostname=sprout.default.svc.cluster.local + + # Keys + signup_key=secret + turn_workaround=secret + ellis_api_key=secret + ellis_cookie_key=secret + + reduce_cassandra_mem_usage=Y + scscf_uri="sip:sprout.default.svc.cluster.local:5054;transport=tcp" + icscf_uri="sip:sprout.default.svc.cluster.local:5052;transport=tcp" + signaling_dns_server=10.123.240.10 + +Cassandra + + root@cassandra-4015560737-f3241:~# cat /etc/cassandra/cassandra.yaml + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + # + # WARNING - THIS FILE IS GENERATED BY ETCD AND SHOULD NOT BE EDITED DIRECTLY + # + # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + authenticator: AllowAllAuthenticator + authorizer: AllowAllAuthorizer + auto_snapshot: true + batch_size_warn_threshold_in_kb: 5 + batchlog_replay_throttle_in_kb: 1024 + broadcast_rpc_address: 10.120.8.7 + cas_contention_timeout_in_ms: 1000 + client_encryption_options: {enabled: false, keystore: conf/.keystore, keystore_password: cassandra, + optional: false} + cluster_name: Test Cluster + column_index_size_in_kb: 64 + commit_failure_policy: stop + commitlog_directory: /var/lib/cassandra/commitlog + commitlog_segment_size_in_mb: 32 + commitlog_sync: periodic + commitlog_sync_period_in_ms: 10000 + compaction_large_partition_warning_threshold_mb: 100 + compaction_throughput_mb_per_sec: 16 + concurrent_counter_writes: 32 + concurrent_reads: 32 + concurrent_writes: 32 + counter_cache_save_period: 7200 + counter_cache_size_in_mb: null + counter_write_request_timeout_in_ms: 5000 + cross_node_timeout: false + data_file_directories: [/var/lib/cassandra/data] + disk_failure_policy: stop + dynamic_snitch_badness_threshold: 0.1 + dynamic_snitch_reset_interval_in_ms: 600000 + dynamic_snitch_update_interval_in_ms: 100 + endpoint_snitch: GossipingPropertyFileSnitch + hinted_handoff_enabled: true + hinted_handoff_throttle_in_kb: 1024 + incremental_backups: false + index_summary_capacity_in_mb: null + index_summary_resize_interval_in_minutes: 60 + inter_dc_tcp_nodelay: false + internode_compression: all + key_cache_save_period: 14400 + key_cache_size_in_mb: null + listen_address: 10.120.8.7 + max_hint_window_in_ms: 10800000 + max_hints_delivery_threads: 2 + memtable_allocation_type: heap_buffers + native_transport_port: 9042 + num_tokens: 256 + partitioner: org.apache.cassandra.dht.Murmur3Partitioner + permissions_validity_in_ms: 2000 + range_request_timeout_in_ms: 10000 + read_request_timeout_in_ms: 400 + request_scheduler: org.apache.cassandra.scheduler.NoScheduler + request_timeout_in_ms: 10000 + row_cache_save_period: 0 + row_cache_size_in_mb: 0 + rpc_address: ::0 + rpc_keepalive: true + rpc_port: 9160 + rpc_server_type: sync + saved_caches_directory: /var/lib/cassandra/saved_caches + seed_provider: + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + - {seeds: 10.120.8.7} + server_encryption_options: {internode_encryption: none, keystore: conf/.keystore, + keystore_password: cassandra, truststore: conf/.truststore, truststore_password: cassandra} + snapshot_before_compaction: false + ssl_storage_port: 7001 + sstable_preemptive_open_interval_in_mb: 50 + start_native_transport: true + start_rpc: true + storage_port: 7000 + thrift_framed_transport_size_in_mb: 15 + tombstone_failure_threshold: 100000 + tombstone_warn_threshold: 1000 + trickle_fsync: false + trickle_fsync_interval_in_kb: 10240 + truncate_request_timeout_in_ms: 60000 + unlogged_batch_across_partitions_warn_threshold: 10 + write_request_timeout_in_ms: 2000 + +### Test + +![make a call](./call-from-xlite-to-zoiper.png "test") + + diff --git a/hack/kubernetes-clearwater/clearwater/call-from-xlite-to-zoiper.png b/hack/kubernetes-clearwater/clearwater/call-from-xlite-to-zoiper.png new file mode 100755 index 0000000..224bd1b Binary files /dev/null and b/hack/kubernetes-clearwater/clearwater/call-from-xlite-to-zoiper.png differ diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/Makefile new file mode 100755 index 0000000..373aba4 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/Makefile @@ -0,0 +1,80 @@ +IMG_NS ?= docker.io/tangfeixiong + +IMG_TAG ?= $(shell date +%y%m%d%H%M).gitrev-$(shell git rev-parse --short=7 HEAD) + +BASE_NAME ?= clearwater-base-onbuild +BONO_NAME ?= clearwater-bono +ELLIS_NAME ?= clearwater-ellis +HOMER_NAME ?= clearwater-homer +HOMESTEAD_NAME ?= clearwater-homestead +RALF_NAME ?= clearwater-ralf +SPROUT_NAME ?= clearwater-sprout +CASSANDRA_NAME ?= clearwater-cassandra +CHRONOS_NAME ?= clearwater-chronos +MEMCACHED_NAME ?= clearwater-memcached + +CTX_ROOT := ../../../.. + +DOCKER_BIN := $(shell which docker) +CP_BIN := $(shell which cp) +SED_BIN := $(shell which sed) + +COMPONENTS := bono ellis homer homestead ralf sprout cassandra chronos memcached + +.PHONY: all base $(COMPONENTS) + +all: $(COMPONENTS) + +base : CTX := $(CTX_ROOT)/base +base : IMG := $(IMG_NS)/$(BASE_NAME):$(IMG_TAG) +base : + $(CP_BIN) $@/Dockerfile.$@ $(CTX)/ + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.$@ $(CTX) + +$(COMPONENTS) : | base + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" $@/Dockerfile.$@ > $(CTX_ROOT)/$@/Dockerfile.$@ + $(DOCKER_BIN) build --tag=$(IMG_NS)/$(value $(shell echo $@ | tr a-z A-Z)_NAME):$(IMG_TAG) --file=$(CTX_ROOT)/$@/Dockerfile.$@ $(CTX_ROOT)/$@ + +# Deprecated + +.PHONY: cw-bono +cw-bono : TGT := bono +cw-bono : IMG := $(IMG_NS)/$(BONO_NAME):$(IMG_TAG) +cw-bono : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" $(TGT)/Dockerfile > $(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) $(CTX_ROOT)/$(TGT) + +.PHONY: cw-ellis +cw-ellis : TGT := ellis +cw-ellis : IMG := $(IMG_NS)/$(ELLIS_NAME):$(IMG_TAG) +cw-ellis : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" $(TGT)/Dockerfile > $(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) $(CTX_ROOT)/$(TGT) + +.PHONY: homer +cw-homer : TGT := homer +cw-homer : IMG := $(IMG_NS)/$(HOMER_NAME):$(IMG_TAG) +cw-homer : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" $(TGT)/Dockerfile > $(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) $(CTX_ROOT)/$(TGT) + +.PHONY: homestead +cw-homestead : TGT := homestead +cw-homestead : IMG := $(IMG_NS)/$(HOMESTEAD_NAME):$(IMG_TAG) +cw-homestead : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" $(TGT)/Dockerfile > $(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) $(CTX_ROOT)/$(TGT) + +.PHONY: ralf +cw-ralf : TGT := ralf +cw-ralf : IMG := $(IMG_NS)/$(RALF_NAME):$(IMG_TAG) +cw-ralf : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" $(TGT)/Dockerfile > $(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) $(CTX_ROOT)/$(TGT) + +.PHONY: sprout +cw-sprout : TGT := sprout +cw-sprout : IMG := $(IMG_NS)/$(SPROUT_NAME):$(IMG_TAG) +cw-sprout : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" $(TGT)/Dockerfile > $(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX_ROOT)/$(TGT)/Dockerfile.$(TGT) $(CTX_ROOT)/$(TGT) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/alt_ops/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/alt_ops/Makefile new file mode 100755 index 0000000..3c4804d --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/alt_ops/Makefile @@ -0,0 +1,84 @@ +IMG_NS ?= 10.64.33.81:5000/admin + +IMG_TAG ?= latest + +BASE_NAME ?= clearwater_base-onbuild +BONO_NAME ?= clearwater_bono +ELLIS_NAME ?= clearwater_ellis +HOMER_NAME ?= clearwater_homer +HOMESTEAD_NAME ?= clearwater_homestead +RALF_NAME ?= clearwater_ralf +SPROUT_NAME ?= clearwater_sprout +CASSANDRA_NAME ?= clearwater_cassandra +CHRONOS_NAME ?= clearwater_chronos +MEMCACHED_NAME ?= clearwater_memcached + +CTX_ROOT := ../../../../.. + +DOCKER_BIN := $(shell which docker) +CP_BIN := $(shell which cp) +SED_BIN := $(shell which sed) + +.PHONY: all base bono ellis homer homestead ralf sprout cassandra chronos memcached + +all: base bono ellis homer homestead ralf sprout cassandra chronos memcached + +base : CTX := $(CTX_ROOT)/base +base : IMG := $(IMG_NS)/$(BASE_NAME):$(IMG_TAG) +base : + $(CP_BIN) ../$@/Dockerfile $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +bono : CTX := $(CTX_ROOT)/bono +bono : IMG := $(IMG_NS)/$(BONO_NAME):$(IMG_TAG) +bono : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +ellis : CTX := $(CTX_ROOT)/ellis +ellis : IMG := $(IMG_NS)/$(ELLIS_NAME):$(IMG_TAG) +ellis : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +homer : CTX := $(CTX_ROOT)/homer +homer : IMG := $(IMG_NS)/$(HOMER_NAME):$(IMG_TAG) +homer : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +homestead : CTX := $(CTX_ROOT)/homestead +homestead : IMG := $(IMG_NS)/$(HOMESTEAD_NAME):$(IMG_TAG) +homestead : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +ralf : CTX := $(CTX_ROOT)/ralf +ralf : IMG := $(IMG_NS)/$(RALF_NAME):$(IMG_TAG) +ralf : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +sprout : CTX := $(CTX_ROOT)/sprout +sprout : IMG := $(IMG_NS)/$(SPROUT_NAME):$(IMG_TAG) +sprout : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +cassandra : CTX := $(CTX_ROOT)/cassandra +cassandra : IMG := $(IMG_NS)/$(CASSANDRA_NAME):$(IMG_TAG) +cassandra : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +chronos : CTX := $(CTX_ROOT)/chronos +chronos : IMG := $(IMG_NS)/$(CHRONOS_NAME):$(IMG_TAG) +chronos : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) + +memcached : CTX := $(CTX_ROOT)/memcached +memcached : IMG := $(IMG_NS)/$(MEMCACHED_NAME):$(IMG_TAG) +memcached : + $(SED_BIN) "s%^FROM .*%FROM $(IMG_NS)/$(BASE_NAME):$(IMG_TAG)%" ../$@/Dockerfile > $(CTX)/Dockerfile.tmp + $(DOCKER_BIN) build --tag=$(IMG) --file=$(CTX)/Dockerfile.tmp $(CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/base/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/base/Dockerfile new file mode 100755 index 0000000..eafbc0a --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/base/Dockerfile @@ -0,0 +1,80 @@ +FROM ubuntu:14.04 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ARG UBUNTU_REPO_MIRROR="http://mirrors.aliyun.com/" +ARG CW_NGV_REPO_MIRROR="http://192.168.1.100:48080/16-mirror/ubuntu/trusty/apt-mirror/mirror/repo.cw-ngv.com/stable" +ARG CW_NGV_REPO_GPG_KEY_URL="http://192.168.1.100:48080/16-mirror/ubuntu/trusty/apt-mirror/gpg-key/http%253A%252F%252Frepo.cw-ngv.com/repo_key" + +ADD sysctl /sbin/sysctl +ADD *supervisord.conf /etc/supervisor/conf.d/ +ADD liveness.sh /usr/share/kubernetes/ + +RUN ( [ -n $UBUNTU_REPO_MIRROR ] \ + && sed -i "s%http://archive.ubuntu.com/%$UBUNTU_REPO_MIRROR%g;s%http://security.ubuntu.com/%$UBUNTU_REPO_MIRROR%g" /etc/apt/sources.list ) \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + openssh-server \ + supervisor \ + curl \ + && mkdir -p /var/run/sshd /var/log/supervisor \ + && echo 'root:root' | chpasswd \ + && sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config \ + && locale-gen en_US.UTF-8 \ + && sed -e 's/\#\(precedence ::ffff:0:0\/96 100\)/\1/g' -i /etc/gai.conf \ + && ( [ -n $CW_NGV_REPO_MIRROR ] \ + && echo deb $CW_NGV_REPO_MIRROR binary/ > /etc/apt/sources.list.d/clearwater.list \ + || echo deb http://repo.cw-ngv.com/stable binary/ > /etc/apt/sources.list.d/clearwater.list ) \ + && ( [ -n $CW_NGV_REPO_GPG_KEY_URL ] \ + && curl -sL $CW_NGV_REPO_GPG_KEY_URL | apt-key add - \ + || curl -L http://repo.cw-ngv.com/repo_key | apt-key add - ) \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + clearwater-infrastructure \ + clearwater-auto-config-docker \ + clearwater-management \ + && sed -i 's/#user=.*/user=root/' /etc/dnsmasq.conf \ + && sed -i 's/\(mv \/tmp\/hosts\.\$\$.*\)/# \1/' /usr/share/clearwater/infrastructure/scripts/1hosts \ + && sed -i 's/ > \/etc\/hostname//;s/\(hostname \$(cat.*\)/# \1/;s/\(mv \/tmp\/hosts\.\$\$.*\)/# \1/' /usr/share/clearwater/infrastructure/scripts/hostname \ + && /etc/init.d/clearwater-auto-config-docker restart \ + && /etc/init.d/clearwater-infrastructure restart \ + && mv /etc/supervisor/conf.d/clearwater-infrastructure.supervisord.conf /etc/supervisor/conf.d/clearwater-infrastructure.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && apt-get install -y netcat-openbsd \ + && sed -i 's/SCTP_streams.*/No_SCTP;/' /usr/share/clearwater/bin/generic_create_diameterconf \ + && sed -i 's/scscf.sprout/sprout/' /etc/init.d/clearwater-auto-config-docker \ + + && sed -i 's/^\s*sprout_hostname=sprout$/ sprout_hostname=sprout.default.svc.cluster.local/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\s*chronos_hostname=chronos$/ chronos_hostname=chronos.default.svc.cluster.local/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\s*cassandra_hostname=cassandra$/ cassandra_hostname=cassandra.default.svc.cluster.local/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\s*hs_hostname=homestead:8888$/ hs_hostname=homestead.default.svc.cluster.local:8888/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\s*hs_provisioning_hostname=homestead:8889$/ hs_provisioning_hostname=homestead.default.svc.cluster.local:8889/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\s*xdms_hostname=homer:7888$/ xdms_hostname=homer.default.svc.cluster.local:7888/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\s*upstream_hostname=sprout$/ upstream_hostname=sprout.default.svc.cluster.local/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\s*ralf_hostname=ralf:10888$/ ralf_hostname=ralf.default.svc.cluster.local:10888/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/home_domain="example.com"/home_domain="default.svc.cluster.local"/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+sprout_hostname=.*\)/\1\n icscf_uri="sip:\$sprout_hostname:5052;transport=tcp"/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+sprout_hostname=.*\)/\1\n scscf_uri="sip:\$sprout_hostname:5054;transport=tcp"/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+s\/^sprout_hostname=.*\)/\1\n s\/^icscf_uri=.*\/icscf_uri="'\''$icscf_uri'\''"\//' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+s\/^sprout_hostname=.*\)/\1\n s\/^scscf_uri=.*\/scscf_uri="'\''$scscf_uri'\''"\//' /etc/init.d/clearwater-auto-config-docker \ + ### echo "scscf_uri=\"sip:sprout:5054;transport=tcp\"" >> /etc/clearwater/shared_config + ### echo "icscf_uri=\"sip:sprout:5052;transport=tcp\"" >> /etc/clearwater/shared_config + && echo "scscf_uri=\"sip:sprout.default.svc.cluster.local:5054;transport=tcp\"" >> /etc/clearwater/shared_config \ + && echo "icscf_uri=\"sip:sprout.default.svc.cluster.local:5052;transport=tcp\"" >> /etc/clearwater/shared_config \ + && sed -i 's/home_domain=example.com/home_domain=default.svc.cluster.local/' /etc/clearwater/shared_config \ + && sed -i 's/sprout_hostname=sprout/sprout_hostname=sprout.default.svc.cluster.local/' /etc/clearwater/shared_config \ + && sed -i 's/hs_hostname=homestead:8888/hs_hostname=homestead.default.svc.cluster.local:8888/' /etc/clearwater/shared_config \ + && sed -i 's/hs_provisioning_hostname=homestead:8889/hs_provisioning_hostname=homestead.default.svc.cluster.local:8889/' /etc/clearwater/shared_config \ + && sed -i 's/xdms_hostname=homer:7888/xdms_hostname=homer.default.svc.cluster.local:7888/' /etc/clearwater/shared_config \ + && sed -i 's/ralf_hostname=ralf:10888/ralf_hostname=ralf.default.svc.cluster.local:10888/' /etc/clearwater/shared_config \ + && sed -i 's/chronos_hostname=chronos/chronos_hostname=chronos.default.svc.cluster.local/' /etc/clearwater/shared_config \ + && sed -i 's/cassandra_hostname=cassandra/cassandra_hostname=cassandra.default.svc.cluster.local/' /etc/clearwater/shared_config \ + && rm -rf /var/lib/apt/lists/* + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +ONBUILD USER root +ONBUILD EXPOSE 22 +ONBUILD CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/base/Dockerfile.base b/hack/kubernetes-clearwater/clearwater/docker-files/base/Dockerfile.base new file mode 100755 index 0000000..280363c --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/base/Dockerfile.base @@ -0,0 +1,55 @@ +FROM ubuntu:14.04 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ARG UBUNTU_REPO_MIRROR="http://mirrors.aliyun.com/" +ARG CW_NGV_REPO_MIRROR="http://10.64.33.1:48080/16-mirror/ubuntu/trusty/apt-mirror/mirror/repo.cw-ngv.com/stable" +ARG CW_NGV_REPO_GPG_KEY_URL="http://10.64.33.1:48080/16-mirror/ubuntu/trusty/apt-mirror/gpg-key/http%253A%252F%252Frepo.cw-ngv.com/repo_key" + +ADD *supervisord.conf /etc/supervisor/conf.d/ +ADD sysctl /sbin/sysctl +ADD liveness.sh /usr/share/kubernetes/ + +RUN ( [ -n $UBUNTU_REPO_MIRROR ] \ + && sed -i "s%http://archive.ubuntu.com/%$UBUNTU_REPO_MIRROR%g;s%http://security.ubuntu.com/%$UBUNTU_REPO_MIRROR%g" /etc/apt/sources.list ) \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server supervisor curl \ + && mkdir -p /var/run/sshd /var/log/supervisor \ + && echo 'root:root' | chpasswd \ + && sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config \ + && locale-gen en_US.UTF-8 \ + && sed -e 's/\#\(precedence ::ffff:0:0\/96 100\)/\1/g' -i /etc/gai.conf \ + && ( [ -n $CW_NGV_REPO_MIRROR ] \ + && echo deb $CW_NGV_REPO_MIRROR binary/ > /etc/apt/sources.list.d/clearwater.list \ + || echo deb http://repo.cw-ngv.com/stable binary/ > /etc/apt/sources.list.d/clearwater.list ) \ + && ( [ -n $CW_NGV_REPO_GPG_KEY_URL ] \ + && curl -sL $CW_NGV_REPO_GPG_KEY_URL | apt-key add - \ + || curl -L http://repo.cw-ngv.com/repo_key | apt-key add - ) \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes \ + clearwater-infrastructure \ + clearwater-auto-config-docker \ + clearwater-management \ + && /etc/init.d/clearwater-auto-config-docker restart \ + && /etc/init.d/clearwater-infrastructure restart \ + && mv /etc/supervisor/conf.d/clearwater-infrastructure.supervisord.conf /etc/supervisor/conf.d/clearwater-infrastructure.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && apt-get install -y netcat-openbsd \ + && sed -i 's/SCTP_streams.*/No_SCTP;/' /usr/share/clearwater/bin/generic_create_diameterconf \ + && sed -i 's/scscf.sprout/sprout/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+sprout_hostname=.*\)/\1\n icscf_uri="sip:\$sprout_hostname:5052;transport=tcp"/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+sprout_hostname=.*\)/\1\n scscf_uri="sip:\$sprout_hostname:5054;transport=tcp"/' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+s\/^sprout_hostname=.*\)/\1\n s\/^icscf_uri=.*\/icscf_uri="'\''$icscf_uri'\''"\//' /etc/init.d/clearwater-auto-config-docker \ + && sed -i 's/^\( \+s\/^sprout_hostname=.*\)/\1\n s\/^scscf_uri=.*\/scscf_uri="'\''$scscf_uri'\''"\//' /etc/init.d/clearwater-auto-config-docker \ + && echo "scscf_uri=\"sip:sprout:5054;transport=tcp\"" >> /etc/clearwater/shared_config \ + && echo "icscf_uri=\"sip:sprout:5052;transport=tcp\"" >> /etc/clearwater/shared_config \ + && rm -rf /var/lib/apt/lists/* + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +ONBUILD USER root +ONBUILD EXPOSE 22 +ONBUILD CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] + diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/base/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/base/Makefile new file mode 100755 index 0000000..2306da4 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/base/Makefile @@ -0,0 +1,12 @@ +IMG_NAME = docker.io/tangfeixiong/clearwater-base-onbuild + +IMG_TAG = $(shell date +%y%m%d%H%M).gitrev-$(shell git rev-parse --short=7 HEAD) + +DOCKER_BIN := $(shell which docker) + +BLD_CTX := ../../../../../base + +.PHONY: all +all: + $(shell cp Dockerfile $(BLD_CTX)/Dockerfile.1) + $(DOCKER_BIN) build --tag=$(IMG_NAME):$(IMG_TAG) --file=$(BLD_CTX)/Dockerfile.1 $(BLD_CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/bono/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/bono/Dockerfile new file mode 100755 index 0000000..9e2ac3d --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/bono/Dockerfile @@ -0,0 +1,21 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + bono \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/bono \ + ### apt-get update + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + restund \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/restund \ + && mv /etc/supervisor/conf.d/bono.supervisord.conf /etc/supervisor/conf.d/bono.conf \ + && mv /etc/supervisor/conf.d/restund.supervisord.conf /etc/supervisor/conf.d/restund.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 3478 3478/udp 5058 5060 5060/udp 5062 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/bono/Dockerfile.bono b/hack/kubernetes-clearwater/clearwater/docker-files/bono/Dockerfile.bono new file mode 100755 index 0000000..0df8a4b --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/bono/Dockerfile.bono @@ -0,0 +1,19 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes bono \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/bono \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes restund \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/restund \ + && mv /etc/supervisor/conf.d/bono.supervisord.conf /etc/supervisor/conf.d/bono.conf \ + && mv /etc/supervisor/conf.d/restund.supervisord.conf /etc/supervisor/conf.d/restund.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 3478 3478/udp 5058 5060 5060/udp 5062 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/bono/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/bono/Makefile new file mode 100755 index 0000000..3665926 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/bono/Makefile @@ -0,0 +1,12 @@ +IMG_NAME = docker.io/tangfeixiong/clearwater-bono + +IMG_TAG = 1.0 + +DOCKER_BIN := $(shell which docker) + +BLD_CTX := ../../../../../bono + +.PHONY: all +all: + $(shell cp Dockerfile $(BLD_CTX)/Dockerfile.1) + $(DOCKER_BIN) build --tag=$(IMG_NAME):$(IMG_TAG) --file=$(BLD_CTX)/Dockerfile.1 $(BLD_CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/cassandra/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/cassandra/Dockerfile new file mode 100755 index 0000000..818394f --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/cassandra/Dockerfile @@ -0,0 +1,19 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# inspired by: maintainers@projectclearwater.org + +COPY *supervisord.conf /etc/supervisor/conf.d/ +COPY start_cassandra.sh /usr/bin/start_cassandra.sh + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + homestead-cassandra \ + homestead-prov-cassandra \ + homer-cassandra \ + && sed -e 's/-c cassandra/-c root/g' -i /etc/init.d/cassandra \ + && mv /etc/supervisor/conf.d/cassandra.supervisord.conf /etc/supervisor/conf.d/cassandra.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 9160 7001 9142 7000 9042 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/cassandra/Dockerfile.cassandra b/hack/kubernetes-clearwater/clearwater/docker-files/cassandra/Dockerfile.cassandra new file mode 100755 index 0000000..b20559f --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/cassandra/Dockerfile.cassandra @@ -0,0 +1,17 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes homestead-cassandra homestead-prov-cassandra homer-cassandra \ + && sed -e 's/-c cassandra/-c root/g' -i /etc/init.d/cassandra \ + && mv /etc/supervisor/conf.d/cassandra.supervisord.conf /etc/supervisor/conf.d/cassandra.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +COPY start_cassandra.sh /usr/bin/start_cassandra.sh + +EXPOSE 9160 7001 9142 7000 9042 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/chronos/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/chronos/Dockerfile new file mode 100755 index 0000000..451d02c --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/chronos/Dockerfile @@ -0,0 +1,16 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + chronos \ + && mv /etc/supervisor/conf.d/snmpd.supervisord.conf /etc/supervisor/conf.d/snmpd.conf \ + && mv /etc/supervisor/conf.d/chronos.supervisord.conf /etc/supervisor/conf.d/chronos.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 7253 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/chronos/Dockerfile.chronos b/hack/kubernetes-clearwater/clearwater/docker-files/chronos/Dockerfile.chronos new file mode 100755 index 0000000..6a84c71 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/chronos/Dockerfile.chronos @@ -0,0 +1,15 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes chronos \ + && mv /etc/supervisor/conf.d/snmpd.supervisord.conf /etc/supervisor/conf.d/snmpd.conf \ + && mv /etc/supervisor/conf.d/chronos.supervisord.conf /etc/supervisor/conf.d/chronos.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 7253 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Dockerfile new file mode 100755 index 0000000..4a63bf3 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Dockerfile @@ -0,0 +1,23 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + mysql-server \ + && /etc/init.d/mysql start \ + ### apt-get update + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + ellis \ + && /etc/init.d/mysql start \ + && /usr/share/clearwater/ellis/env/bin/python /usr/share/clearwater/ellis/src/metaswitch/ellis/tools/create_numbers.py --start 6505550000 --count 1000 \ + && mv /etc/supervisor/conf.d/ellis.supervisord.conf /etc/supervisor/conf.d/ellis.conf \ + && mv /etc/supervisor/conf.d/mysql.supervisord.conf /etc/supervisor/conf.d/mysql.conf \ + && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 80 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Dockerfile.ellis b/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Dockerfile.ellis new file mode 100755 index 0000000..ec26158 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Dockerfile.ellis @@ -0,0 +1,21 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes mysql-server \ + && /etc/init.d/mysql start \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes ellis \ + && /etc/init.d/mysql start \ + && /usr/share/clearwater/ellis/env/bin/python /usr/share/clearwater/ellis/src/metaswitch/ellis/tools/create_numbers.py --start 6505550000 --count 1000 \ + && mv /etc/supervisor/conf.d/ellis.supervisord.conf /etc/supervisor/conf.d/ellis.conf \ + && mv /etc/supervisor/conf.d/mysql.supervisord.conf /etc/supervisor/conf.d/mysql.conf \ + && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 80 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Makefile new file mode 100755 index 0000000..307f7ed --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/ellis/Makefile @@ -0,0 +1,12 @@ +IMG_NAME = docker.io/tangfeixiong/clearwater-ellis + +IMG_TAG = 1.0 + +DOCKER_BIN := $(shell which docker) + +BLD_CTX := ../../../../../ellis + +.PHONY: all +all: + $(shell cp Dockerfile $(BLD_CTX)/Dockerfile.1) + $(DOCKER_BIN) build --tag=$(IMG_NAME):$(IMG_TAG) --file=$(BLD_CTX)/Dockerfile.1 $(BLD_CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/homer/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/homer/Dockerfile new file mode 100755 index 0000000..6c29909 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/homer/Dockerfile @@ -0,0 +1,16 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + homer \ + && mv /etc/supervisor/conf.d/homer.supervisord.conf /etc/supervisor/conf.d/homer.conf \ + && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 7888 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/homer/Dockerfile.homer b/hack/kubernetes-clearwater/clearwater/docker-files/homer/Dockerfile.homer new file mode 100755 index 0000000..e56bf12 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/homer/Dockerfile.homer @@ -0,0 +1,15 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes homer \ + && mv /etc/supervisor/conf.d/homer.supervisord.conf /etc/supervisor/conf.d/homer.conf \ + && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 7888 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/homer/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/homer/Makefile new file mode 100755 index 0000000..8bcc9c9 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/homer/Makefile @@ -0,0 +1,12 @@ +IMG_NAME = docker.io/tangfeixiong/clearwater-homer + +IMG_TAG = 1.0 + +DOCKER_BIN := $(shell which docker) + +BLD_CTX := ../../../../../homer + +.PHONY: all +all: + $(shell cp Dockerfile $(BLD_CTX)/Dockerfile.1) + $(DOCKER_BIN) build --tag=$(IMG_NAME):$(IMG_TAG) --file=$(BLD_CTX)/Dockerfile.1 $(BLD_CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Dockerfile new file mode 100755 index 0000000..c595486 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Dockerfile @@ -0,0 +1,20 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + homestead \ + homestead-prov \ + clearwater-prov-tools \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \1/g' -i /etc/init.d/homestead \ + && mv /etc/supervisor/conf.d/homestead.supervisord.conf /etc/supervisor/conf.d/homestead.conf \ + && mv /etc/supervisor/conf.d/homestead-prov.supervisord.conf /etc/supervisor/conf.d/homestead-prov.conf \ + && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 8888 8889 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Dockerfile.homestead b/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Dockerfile.homestead new file mode 100755 index 0000000..2bde067 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Dockerfile.homestead @@ -0,0 +1,17 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes homestead homestead-prov clearwater-prov-tools \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \1/g' -i /etc/init.d/homestead \ + && mv /etc/supervisor/conf.d/homestead.supervisord.conf /etc/supervisor/conf.d/homestead.conf \ + && mv /etc/supervisor/conf.d/homestead-prov.supervisord.conf /etc/supervisor/conf.d/homestead-prov.conf \ + && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 8888 8889 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Makefile new file mode 100755 index 0000000..ff01c05 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/homestead/Makefile @@ -0,0 +1,12 @@ +IMG_NAME = docker.io/tangfeixiong/clearwater-homestead + +IMG_TAG = 1.0 + +DOCKER_BIN := $(shell which docker) + +BLD_CTX := ../../../../../homestead + +.PHONY: all +all: + $(shell cp Dockerfile $(BLD_CTX)/Dockerfile.1) + $(DOCKER_BIN) build --tag=$(IMG_NAME):$(IMG_TAG) --file=$(BLD_CTX)/Dockerfile.1 $(BLD_CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/memcached/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/memcached/Dockerfile new file mode 100755 index 0000000..049930b --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/memcached/Dockerfile @@ -0,0 +1,15 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + clearwater-memcached \ + && mv /etc/supervisor/conf.d/memcached.supervisord.conf /etc/supervisor/conf.d/memcached.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 11211 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/memcached/Dockerfile.memcached b/hack/kubernetes-clearwater/clearwater/docker-files/memcached/Dockerfile.memcached new file mode 100755 index 0000000..eacc4a1 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/memcached/Dockerfile.memcached @@ -0,0 +1,14 @@ +# FROM clearwater/base-onbuild +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes clearwater-memcached \ + && mv /etc/supervisor/conf.d/memcached.supervisord.conf /etc/supervisor/conf.d/memcached.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 11211 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Dockerfile new file mode 100755 index 0000000..fe4489c --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Dockerfile @@ -0,0 +1,18 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ +COPY plugins/* /usr/share/clearwater/clearwater-cluster-manager/plugins/ +ADD reload_memcached_users /usr/share/clearwater/bin/reload_memcached_users + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + ralf \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/ralf \ + && mv /etc/supervisor/conf.d/ralf.supervisord.conf /etc/supervisor/conf.d/ralf.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 10888 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Dockerfile.ralf b/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Dockerfile.ralf new file mode 100755 index 0000000..079ab8a --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Dockerfile.ralf @@ -0,0 +1,18 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes ralf \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/ralf \ + && mv /etc/supervisor/conf.d/ralf.supervisord.conf /etc/supervisor/conf.d/ralf.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +ADD plugins/* /usr/share/clearwater/clearwater-cluster-manager/plugins/ +ADD reload_memcached_users /usr/share/clearwater/bin/reload_memcached_users + +EXPOSE 10888 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Makefile new file mode 100755 index 0000000..3f50472 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/ralf/Makefile @@ -0,0 +1,12 @@ +IMG_NAME = docker.io/tangfeixiong/clearwater-ralf + +IMG_TAG = 1.0 + +DOCKER_BIN := $(shell which docker) + +BLD_CTX := ../../../../../ralf + +.PHONY: all +all: + $(shell cp Dockerfile $(BLD_CTX)/Dockerfile.1) + $(DOCKER_BIN) build --tag=$(IMG_NAME):$(IMG_TAG) --file=$(BLD_CTX)/Dockerfile.1 $(BLD_CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Dockerfile b/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Dockerfile new file mode 100755 index 0000000..047d550 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Dockerfile @@ -0,0 +1,18 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ +COPY plugins/* /usr/share/clearwater/clearwater-cluster-manager/plugins/ +ADD reload_memcached_users /usr/share/clearwater/bin/reload_memcached_users + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes --no-install-recommends \ + sprout \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/sprout \ + && mv /etc/supervisor/conf.d/sprout.supervisord.conf /etc/supervisor/conf.d/sprout.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +EXPOSE 5052 5054 \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Dockerfile.sprout b/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Dockerfile.sprout new file mode 100755 index 0000000..9856ad3 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Dockerfile.sprout @@ -0,0 +1,18 @@ +# FROM clearwater/base +FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 +MAINTAINER tangfeixiong +# Inspired by: maintainers@projectclearwater.org + +ADD *supervisord.conf /etc/supervisor/conf.d/ + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes sprout \ + && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/sprout \ + && mv /etc/supervisor/conf.d/sprout.supervisord.conf /etc/supervisor/conf.d/sprout.conf \ + && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf \ + && rm -rf /var/lib/apt/lists/* + +ADD plugins/* /usr/share/clearwater/clearwater-cluster-manager/plugins/ +ADD reload_memcached_users /usr/share/clearwater/bin/reload_memcached_users + +EXPOSE 5052 5054 diff --git a/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Makefile b/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Makefile new file mode 100755 index 0000000..9d0b913 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docker-files/sprout/Makefile @@ -0,0 +1,12 @@ +IMG_NAME = docker.io/tangfeixiong/clearwater-sprout + +IMG_TAG = 1.0 + +DOCKER_BIN := $(shell which docker) + +BLD_CTX := ../../../../../sprout + +.PHONY: all +all: + $(shell cp Dockerfile $(BLD_CTX)/Dockerfile.1) + $(DOCKER_BIN) build --tag=$(IMG_NAME):$(IMG_TAG) --file=$(BLD_CTX)/Dockerfile.1 $(BLD_CTX) diff --git a/hack/kubernetes-clearwater/clearwater/docs/base-docker-build.md b/hack/kubernetes-clearwater/clearwater/docs/base-docker-build.md new file mode 100755 index 0000000..04d213b --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docs/base-docker-build.md @@ -0,0 +1,1453 @@ +Tables of content +------------------- + +* A local `repo.cw-ngv.com` mirror + +* Customized `Dockerfile` for domestic `archive.ubuntu.com` mirror + +* Invoke `docker build` with `make` + +Content + + tangf@DESKTOP-H68OQDV /cygdrive/f/16-mirror/apt-mirror/trusty + $ gofileserver.exe + Listening at :48080 + + + [tangfx@localhost base-docker]$ pwd + /go/src/github.com/Metaswitch/clearwater-docker/hack/kubernetes-clearwater/clearwater/base-docker + + [tangfx@localhost base-docker]$ ls + Dockerfile Makefile + + + [tangfx@localhost cw-docker]$ make base IMG_TAG=1611180614.gitrev-c2da9c9 + /usr/bin/cp base/Dockerfile ../../../base/Dockerfile.tangfx + /usr/bin/docker build --tag=docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 --file=../../../base/Dockerfile.tangfx ../../../base + Sending build context to Docker daemon 10.24 kB + Step 1 : FROM ubuntu:14.04 + ---> 1e0c3dd64ccd + Step 2 : MAINTAINER tangfeixiong + ---> Using cache + ---> 38fb803106e2 + Step 3 : ENV UBUNTU_REPO_MIRROR "http://mirrors.aliyun.com/" CW_NGV_REPO_MIRROR "http://10.64.33.1:48080/mirror/repo.cw-ngv.com" CW_NGV_REPO_GPG_KEY_URL http://10.64.33.1:48080/gpg-key/http%253A%252F%252Frepo.cw-ngv.com + ---> Using cache + ---> 296c3573b4e7 + Step 4 : ADD sysctl /sbin/sysctl + ---> Using cache + ---> 02e9c7bd6164 + Step 5 : ADD *supervisord.conf /etc/supervisor/conf.d/ + ---> Using cache + ---> 2295a4ac32d6 + Step 6 : RUN ( [ -n $UBUNTU_REPO_MIRROR ] && sed -i "s%http://archive.ubuntu.com/%$UBUNTU_REPO_MIRROR%g;s%http://security.ubuntu.com/%$UBUNTU_REPO_MIRROR%g" /etc/apt/sources.list ) && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y openssh-server supervisor curl && mkdir -p /var/run/sshd /var/log/supervisor && echo 'root:root' | chpasswd && sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config && locale-gen en_US.UTF-8 && sed -e 's/\#\(precedence ::ffff:0:0\/96 100\)/\1/g' -i /etc/gai.conf && ( [ -n $CW_NGV_REPO_MIRROR ] && echo deb $CW_NGV_REPO_MIRROR/stable binary/ > /etc/apt/sources.list.d/clearwater.list || echo deb http://repo.cw-ngv.com/stable binary/ > /etc/apt/sources.list.d/clearwater.list ) && ( [ -n $CW_NGV_REPO_GPG_KEY_URL ] && curl -sL $CW_NGV_REPO_MIRROR/repo_key | apt-key add - || curl -L http://repo.cw-ngv.com/repo_key | apt-key add - ) && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes clearwater-infrastructure clearwater-auto-config-docker clearwater-management && /etc/init.d/clearwater-auto-config-docker restart && /etc/init.d/clearwater-infrastructure restart && mv /etc/supervisor/conf.d/clearwater-infrastructure.supervisord.conf /etc/supervisor/conf.d/clearwater-infrastructure.conf && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf && rm -rf /var/lib/apt/lists/* + ---> Running in 7b08f9c8fc13 + Ign http://mirrors.aliyun.com trusty InRelease + Get:1 http://mirrors.aliyun.com trusty-updates InRelease [65.9 kB] + Get:2 http://mirrors.aliyun.com trusty-security InRelease [65.9 kB] + Get:3 http://mirrors.aliyun.com trusty Release.gpg [933 B] + Get:4 http://mirrors.aliyun.com trusty Release [58.5 kB] + Get:5 http://mirrors.aliyun.com trusty-updates/main Sources [476 kB] + Get:6 http://mirrors.aliyun.com trusty-updates/restricted Sources [5921 B] + Get:7 http://mirrors.aliyun.com trusty-updates/universe Sources [214 kB] + Get:8 http://mirrors.aliyun.com trusty-updates/main amd64 Packages [1145 kB] + Get:9 http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages [20.4 kB] + Get:10 http://mirrors.aliyun.com trusty-updates/universe amd64 Packages [502 kB] + Get:11 http://mirrors.aliyun.com trusty-security/main Sources [153 kB] + Get:12 http://mirrors.aliyun.com trusty-security/restricted Sources [4621 B] + Get:13 http://mirrors.aliyun.com trusty-security/universe Sources [54.0 kB] + Get:14 http://mirrors.aliyun.com trusty-security/main amd64 Packages [681 kB] + Get:15 http://mirrors.aliyun.com trusty-security/restricted amd64 Packages [17.0 kB] + Get:16 http://mirrors.aliyun.com trusty-security/universe amd64 Packages [188 kB] + Get:17 http://mirrors.aliyun.com trusty/main Sources [1335 kB] + Get:18 http://mirrors.aliyun.com trusty/restricted Sources [5335 B] + Get:19 http://mirrors.aliyun.com trusty/universe Sources [7926 kB] + Get:20 http://mirrors.aliyun.com trusty/main amd64 Packages [1743 kB] + Get:21 http://mirrors.aliyun.com trusty/restricted amd64 Packages [16.0 kB] + Get:22 http://mirrors.aliyun.com trusty/universe amd64 Packages [7589 kB] + Fetched 22.3 MB in 22s (972 kB/s) + Reading package lists... + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + ca-certificates krb5-locales libasn1-8-heimdal libck-connector0 libcurl3 + libedit2 libgssapi-krb5-2 libgssapi3-heimdal libhcrypto4-heimdal + libheimbase1-heimdal libheimntlm0-heimdal libhx509-5-heimdal libidn11 + libk5crypto3 libkeyutils1 libkrb5-26-heimdal libkrb5-3 libkrb5support0 + libldap-2.4-2 libpython-stdlib libpython2.7-minimal libpython2.7-stdlib + libroken18-heimdal librtmp0 libsasl2-2 libsasl2-modules libsasl2-modules-db + libwind0-heimdal libwrap0 libx11-6 libx11-data libxau6 libxcb1 libxdmcp6 + libxext6 libxmuu1 ncurses-term openssh-client openssh-sftp-server openssl + python python-chardet python-meld3 python-minimal python-pkg-resources + python-requests python-six python-urllib3 python2.7 python2.7-minimal + ssh-import-id tcpd wget xauth + Suggested packages: + krb5-doc krb5-user libsasl2-modules-otp libsasl2-modules-ldap + libsasl2-modules-sql libsasl2-modules-gssapi-mit + libsasl2-modules-gssapi-heimdal ssh-askpass libpam-ssh keychain monkeysphere + rssh molly-guard ufw python-doc python-tk python-distribute + python-distribute-doc python2.7-doc binutils binfmt-support + The following NEW packages will be installed: + ca-certificates curl krb5-locales libasn1-8-heimdal libck-connector0 + libcurl3 libedit2 libgssapi-krb5-2 libgssapi3-heimdal libhcrypto4-heimdal + libheimbase1-heimdal libheimntlm0-heimdal libhx509-5-heimdal libidn11 + libk5crypto3 libkeyutils1 libkrb5-26-heimdal libkrb5-3 libkrb5support0 + libldap-2.4-2 libpython-stdlib libpython2.7-minimal libpython2.7-stdlib + libroken18-heimdal librtmp0 libsasl2-2 libsasl2-modules libsasl2-modules-db + libwind0-heimdal libwrap0 libx11-6 libx11-data libxau6 libxcb1 libxdmcp6 + libxext6 libxmuu1 ncurses-term openssh-client openssh-server + openssh-sftp-server openssl python python-chardet python-meld3 + python-minimal python-pkg-resources python-requests python-six + python-urllib3 python2.7 python2.7-minimal ssh-import-id supervisor tcpd + wget xauth + 0 upgraded, 57 newly installed, 0 to remove and 9 not upgraded. + Need to get 9433 kB of archives. + After this operation, 42.4 MB of additional disk space will be used. + Get:1 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libroken18-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [40.0 kB] + Get:2 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libasn1-8-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [161 kB] + Get:3 http://mirrors.aliyun.com/ubuntu/ trusty/main libedit2 amd64 3.1-20130712-2 [86.7 kB] + Get:4 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libkrb5support0 amd64 1.12+dfsg-2ubuntu5.2 [30.7 kB] + Get:5 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libk5crypto3 amd64 1.12+dfsg-2ubuntu5.2 [79.4 kB] + Get:6 http://mirrors.aliyun.com/ubuntu/ trusty/main libkeyutils1 amd64 1.5.6-1 [7318 B] + Get:7 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libkrb5-3 amd64 1.12+dfsg-2ubuntu5.2 [262 kB] + Get:8 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgssapi-krb5-2 amd64 1.12+dfsg-2ubuntu5.2 [114 kB] + Get:9 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libhcrypto4-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [83.9 kB] + Get:10 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libheimbase1-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [28.9 kB] + Get:11 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libwind0-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [47.8 kB] + Get:12 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libhx509-5-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [104 kB] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libkrb5-26-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [196 kB] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libheimntlm0-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [15.2 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgssapi3-heimdal amd64 1.6~git20131207+dfsg-1ubuntu1.1 [89.8 kB] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libidn11 amd64 1.28-1ubuntu2.1 [94.4 kB] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty/main libsasl2-modules-db amd64 2.1.25.dfsg1-17build1 [14.9 kB] + Get:18 http://mirrors.aliyun.com/ubuntu/ trusty/main libsasl2-2 amd64 2.1.25.dfsg1-17build1 [56.5 kB] + Get:19 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libldap-2.4-2 amd64 2.4.31-1+nmu2ubuntu8.3 [153 kB] + Get:20 http://mirrors.aliyun.com/ubuntu/ trusty/main librtmp0 amd64 2.4+20121230.gitdf6c518-1 [57.5 kB] + Get:21 http://mirrors.aliyun.com/ubuntu/ trusty/main libxau6 amd64 1:1.0.8-1 [8376 B] + Get:22 http://mirrors.aliyun.com/ubuntu/ trusty/main libxdmcp6 amd64 1:1.1.1-1 [12.8 kB] + Get:23 http://mirrors.aliyun.com/ubuntu/ trusty/main libxcb1 amd64 1.10-2ubuntu1 [38.0 kB] + Get:24 http://mirrors.aliyun.com/ubuntu/ trusty/main libx11-data all 2:1.6.2-1ubuntu2 [111 kB] + Get:25 http://mirrors.aliyun.com/ubuntu/ trusty/main libx11-6 amd64 2:1.6.2-1ubuntu2 [560 kB] + Get:26 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libxext6 amd64 2:1.3.2-1ubuntu0.0.14.04.1 [28.8 kB] + Get:27 http://mirrors.aliyun.com/ubuntu/ trusty/main libxmuu1 amd64 2:1.1.1-1 [11.0 kB] + Get:28 http://mirrors.aliyun.com/ubuntu/ trusty/main libck-connector0 amd64 0.4.5-3.1ubuntu2 [10.5 kB] + Get:29 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libcurl3 amd64 7.35.0-1ubuntu2.10 [173 kB] + Get:30 http://mirrors.aliyun.com/ubuntu/ trusty/main libwrap0 amd64 7.6.q-25 [46.2 kB] + Get:31 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpython2.7-minimal amd64 2.7.6-8ubuntu0.2 [308 kB] + Get:32 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python2.7-minimal amd64 2.7.6-8ubuntu0.2 [1185 kB] + Get:33 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main openssl amd64 1.0.1f-1ubuntu2.21 [489 kB] + Get:34 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main ca-certificates all 20160104ubuntu0.14.04.1 [190 kB] + Get:35 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main krb5-locales all 1.12+dfsg-2ubuntu5.2 [13.6 kB] + Get:36 http://mirrors.aliyun.com/ubuntu/ trusty/main libsasl2-modules amd64 2.1.25.dfsg1-17build1 [64.3 kB] + Get:37 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main openssh-client amd64 1:6.6p1-2ubuntu2.8 [564 kB] + Get:38 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main wget amd64 1.15-1ubuntu1.14.04.2 [271 kB] + Get:39 http://mirrors.aliyun.com/ubuntu/ trusty/main xauth amd64 1:1.0.7-1ubuntu1 [25.8 kB] + Get:40 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main curl amd64 7.35.0-1ubuntu2.10 [123 kB] + Get:41 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpython2.7-stdlib amd64 2.7.6-8ubuntu0.2 [1869 kB] + Get:42 http://mirrors.aliyun.com/ubuntu/ trusty/main libpython-stdlib amd64 2.7.5-5ubuntu3 [7012 B] + Get:43 http://mirrors.aliyun.com/ubuntu/ trusty/main ncurses-term all 5.9+20140118-1ubuntu1 [243 kB] + Get:44 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main openssh-sftp-server amd64 1:6.6p1-2ubuntu2.8 [34.2 kB] + Get:45 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main openssh-server amd64 1:6.6p1-2ubuntu2.8 [322 kB] + Get:46 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python2.7 amd64 2.7.6-8ubuntu0.2 [196 kB] + Get:47 http://mirrors.aliyun.com/ubuntu/ trusty/main python-minimal amd64 2.7.5-5ubuntu3 [27.5 kB] + Get:48 http://mirrors.aliyun.com/ubuntu/ trusty/main python amd64 2.7.5-5ubuntu3 [134 kB] + Get:49 http://mirrors.aliyun.com/ubuntu/ trusty/main python-chardet all 2.0.1-2build2 [106 kB] + Get:50 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-pkg-resources all 3.3-1ubuntu2 [61.9 kB] + Get:51 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-six all 1.5.2-1ubuntu1 [8238 B] + Get:52 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-urllib3 all 1.7.1-1ubuntu4 [39.5 kB] + Get:53 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-requests all 2.2.1-1ubuntu0.3 [43.1 kB] + Get:54 http://mirrors.aliyun.com/ubuntu/ trusty/main tcpd amd64 7.6.q-25 [23.0 kB] + Get:55 http://mirrors.aliyun.com/ubuntu/ trusty/universe python-meld3 amd64 0.6.10-1 [39.0 kB] + Get:56 http://mirrors.aliyun.com/ubuntu/ trusty/main ssh-import-id all 3.21-0ubuntu1 [9624 B] + Get:57 http://mirrors.aliyun.com/ubuntu/ trusty/universe supervisor all 3.0b2-1 [314 kB] + Extracting templates from packages: 100% + Preconfiguring packages ... + Fetched 9433 kB in 9s (982 kB/s) + Selecting previously unselected package libroken18-heimdal:amd64. + (Reading database ... 11558 files and directories currently installed.) + Preparing to unpack .../libroken18-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libroken18-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libasn1-8-heimdal:amd64. + Preparing to unpack .../libasn1-8-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libasn1-8-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libedit2:amd64. + Preparing to unpack .../libedit2_3.1-20130712-2_amd64.deb ... + Unpacking libedit2:amd64 (3.1-20130712-2) ... + Selecting previously unselected package libkrb5support0:amd64. + Preparing to unpack .../libkrb5support0_1.12+dfsg-2ubuntu5.2_amd64.deb ... + Unpacking libkrb5support0:amd64 (1.12+dfsg-2ubuntu5.2) ... + Selecting previously unselected package libk5crypto3:amd64. + Preparing to unpack .../libk5crypto3_1.12+dfsg-2ubuntu5.2_amd64.deb ... + Unpacking libk5crypto3:amd64 (1.12+dfsg-2ubuntu5.2) ... + Selecting previously unselected package libkeyutils1:amd64. + Preparing to unpack .../libkeyutils1_1.5.6-1_amd64.deb ... + Unpacking libkeyutils1:amd64 (1.5.6-1) ... + Selecting previously unselected package libkrb5-3:amd64. + Preparing to unpack .../libkrb5-3_1.12+dfsg-2ubuntu5.2_amd64.deb ... + Unpacking libkrb5-3:amd64 (1.12+dfsg-2ubuntu5.2) ... + Selecting previously unselected package libgssapi-krb5-2:amd64. + Preparing to unpack .../libgssapi-krb5-2_1.12+dfsg-2ubuntu5.2_amd64.deb ... + Unpacking libgssapi-krb5-2:amd64 (1.12+dfsg-2ubuntu5.2) ... + Selecting previously unselected package libhcrypto4-heimdal:amd64. + Preparing to unpack .../libhcrypto4-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libhcrypto4-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libheimbase1-heimdal:amd64. + Preparing to unpack .../libheimbase1-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libheimbase1-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libwind0-heimdal:amd64. + Preparing to unpack .../libwind0-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libwind0-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libhx509-5-heimdal:amd64. + Preparing to unpack .../libhx509-5-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libhx509-5-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libkrb5-26-heimdal:amd64. + Preparing to unpack .../libkrb5-26-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libkrb5-26-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libheimntlm0-heimdal:amd64. + Preparing to unpack .../libheimntlm0-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libheimntlm0-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libgssapi3-heimdal:amd64. + Preparing to unpack .../libgssapi3-heimdal_1.6~git20131207+dfsg-1ubuntu1.1_amd64.deb ... + Unpacking libgssapi3-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Selecting previously unselected package libidn11:amd64. + Preparing to unpack .../libidn11_1.28-1ubuntu2.1_amd64.deb ... + Unpacking libidn11:amd64 (1.28-1ubuntu2.1) ... + Selecting previously unselected package libsasl2-modules-db:amd64. + Preparing to unpack .../libsasl2-modules-db_2.1.25.dfsg1-17build1_amd64.deb ... + Unpacking libsasl2-modules-db:amd64 (2.1.25.dfsg1-17build1) ... + Selecting previously unselected package libsasl2-2:amd64. + Preparing to unpack .../libsasl2-2_2.1.25.dfsg1-17build1_amd64.deb ... + Unpacking libsasl2-2:amd64 (2.1.25.dfsg1-17build1) ... + Selecting previously unselected package libldap-2.4-2:amd64. + Preparing to unpack .../libldap-2.4-2_2.4.31-1+nmu2ubuntu8.3_amd64.deb ... + Unpacking libldap-2.4-2:amd64 (2.4.31-1+nmu2ubuntu8.3) ... + Selecting previously unselected package librtmp0:amd64. + Preparing to unpack .../librtmp0_2.4+20121230.gitdf6c518-1_amd64.deb ... + Unpacking librtmp0:amd64 (2.4+20121230.gitdf6c518-1) ... + Selecting previously unselected package libxau6:amd64. + Preparing to unpack .../libxau6_1%3a1.0.8-1_amd64.deb ... + Unpacking libxau6:amd64 (1:1.0.8-1) ... + Selecting previously unselected package libxdmcp6:amd64. + Preparing to unpack .../libxdmcp6_1%3a1.1.1-1_amd64.deb ... + Unpacking libxdmcp6:amd64 (1:1.1.1-1) ... + Selecting previously unselected package libxcb1:amd64. + Preparing to unpack .../libxcb1_1.10-2ubuntu1_amd64.deb ... + Unpacking libxcb1:amd64 (1.10-2ubuntu1) ... + Selecting previously unselected package libx11-data. + Preparing to unpack .../libx11-data_2%3a1.6.2-1ubuntu2_all.deb ... + Unpacking libx11-data (2:1.6.2-1ubuntu2) ... + Selecting previously unselected package libx11-6:amd64. + Preparing to unpack .../libx11-6_2%3a1.6.2-1ubuntu2_amd64.deb ... + Unpacking libx11-6:amd64 (2:1.6.2-1ubuntu2) ... + Selecting previously unselected package libxext6:amd64. + Preparing to unpack .../libxext6_2%3a1.3.2-1ubuntu0.0.14.04.1_amd64.deb ... + Unpacking libxext6:amd64 (2:1.3.2-1ubuntu0.0.14.04.1) ... + Selecting previously unselected package libxmuu1:amd64. + Preparing to unpack .../libxmuu1_2%3a1.1.1-1_amd64.deb ... + Unpacking libxmuu1:amd64 (2:1.1.1-1) ... + Selecting previously unselected package libck-connector0:amd64. + Preparing to unpack .../libck-connector0_0.4.5-3.1ubuntu2_amd64.deb ... + Unpacking libck-connector0:amd64 (0.4.5-3.1ubuntu2) ... + Selecting previously unselected package libcurl3:amd64. + Preparing to unpack .../libcurl3_7.35.0-1ubuntu2.10_amd64.deb ... + Unpacking libcurl3:amd64 (7.35.0-1ubuntu2.10) ... + Selecting previously unselected package libwrap0:amd64. + Preparing to unpack .../libwrap0_7.6.q-25_amd64.deb ... + Unpacking libwrap0:amd64 (7.6.q-25) ... + Selecting previously unselected package libpython2.7-minimal:amd64. + Preparing to unpack .../libpython2.7-minimal_2.7.6-8ubuntu0.2_amd64.deb ... + Unpacking libpython2.7-minimal:amd64 (2.7.6-8ubuntu0.2) ... + Selecting previously unselected package python2.7-minimal. + Preparing to unpack .../python2.7-minimal_2.7.6-8ubuntu0.2_amd64.deb ... + Unpacking python2.7-minimal (2.7.6-8ubuntu0.2) ... + Selecting previously unselected package openssl. + Preparing to unpack .../openssl_1.0.1f-1ubuntu2.21_amd64.deb ... + Unpacking openssl (1.0.1f-1ubuntu2.21) ... + Selecting previously unselected package ca-certificates. + Preparing to unpack .../ca-certificates_20160104ubuntu0.14.04.1_all.deb ... + Unpacking ca-certificates (20160104ubuntu0.14.04.1) ... + Selecting previously unselected package krb5-locales. + Preparing to unpack .../krb5-locales_1.12+dfsg-2ubuntu5.2_all.deb ... + Unpacking krb5-locales (1.12+dfsg-2ubuntu5.2) ... + Selecting previously unselected package libsasl2-modules:amd64. + Preparing to unpack .../libsasl2-modules_2.1.25.dfsg1-17build1_amd64.deb ... + Unpacking libsasl2-modules:amd64 (2.1.25.dfsg1-17build1) ... + Selecting previously unselected package openssh-client. + Preparing to unpack .../openssh-client_1%3a6.6p1-2ubuntu2.8_amd64.deb ... + Unpacking openssh-client (1:6.6p1-2ubuntu2.8) ... + Selecting previously unselected package wget. + Preparing to unpack .../wget_1.15-1ubuntu1.14.04.2_amd64.deb ... + Unpacking wget (1.15-1ubuntu1.14.04.2) ... + Selecting previously unselected package xauth. + Preparing to unpack .../xauth_1%3a1.0.7-1ubuntu1_amd64.deb ... + Unpacking xauth (1:1.0.7-1ubuntu1) ... + Selecting previously unselected package curl. + Preparing to unpack .../curl_7.35.0-1ubuntu2.10_amd64.deb ... + Unpacking curl (7.35.0-1ubuntu2.10) ... + Selecting previously unselected package libpython2.7-stdlib:amd64. + Preparing to unpack .../libpython2.7-stdlib_2.7.6-8ubuntu0.2_amd64.deb ... + Unpacking libpython2.7-stdlib:amd64 (2.7.6-8ubuntu0.2) ... + Selecting previously unselected package libpython-stdlib:amd64. + Preparing to unpack .../libpython-stdlib_2.7.5-5ubuntu3_amd64.deb ... + Unpacking libpython-stdlib:amd64 (2.7.5-5ubuntu3) ... + Selecting previously unselected package ncurses-term. + Preparing to unpack .../ncurses-term_5.9+20140118-1ubuntu1_all.deb ... + Unpacking ncurses-term (5.9+20140118-1ubuntu1) ... + Selecting previously unselected package openssh-sftp-server. + Preparing to unpack .../openssh-sftp-server_1%3a6.6p1-2ubuntu2.8_amd64.deb ... + Unpacking openssh-sftp-server (1:6.6p1-2ubuntu2.8) ... + Selecting previously unselected package openssh-server. + Preparing to unpack .../openssh-server_1%3a6.6p1-2ubuntu2.8_amd64.deb ... + Unpacking openssh-server (1:6.6p1-2ubuntu2.8) ... + Selecting previously unselected package python2.7. + Preparing to unpack .../python2.7_2.7.6-8ubuntu0.2_amd64.deb ... + Unpacking python2.7 (2.7.6-8ubuntu0.2) ... + Selecting previously unselected package python-minimal. + Preparing to unpack .../python-minimal_2.7.5-5ubuntu3_amd64.deb ... + Unpacking python-minimal (2.7.5-5ubuntu3) ... + Selecting previously unselected package python. + Preparing to unpack .../python_2.7.5-5ubuntu3_amd64.deb ... + Unpacking python (2.7.5-5ubuntu3) ... + Selecting previously unselected package python-chardet. + Preparing to unpack .../python-chardet_2.0.1-2build2_all.deb ... + Unpacking python-chardet (2.0.1-2build2) ... + Selecting previously unselected package python-pkg-resources. + Preparing to unpack .../python-pkg-resources_3.3-1ubuntu2_all.deb ... + Unpacking python-pkg-resources (3.3-1ubuntu2) ... + Selecting previously unselected package python-six. + Preparing to unpack .../python-six_1.5.2-1ubuntu1_all.deb ... + Unpacking python-six (1.5.2-1ubuntu1) ... + Selecting previously unselected package python-urllib3. + Preparing to unpack .../python-urllib3_1.7.1-1ubuntu4_all.deb ... + Unpacking python-urllib3 (1.7.1-1ubuntu4) ... + Selecting previously unselected package python-requests. + Preparing to unpack .../python-requests_2.2.1-1ubuntu0.3_all.deb ... + Unpacking python-requests (2.2.1-1ubuntu0.3) ... + Selecting previously unselected package tcpd. + Preparing to unpack .../tcpd_7.6.q-25_amd64.deb ... + Unpacking tcpd (7.6.q-25) ... + Selecting previously unselected package python-meld3. + Preparing to unpack .../python-meld3_0.6.10-1_amd64.deb ... + Unpacking python-meld3 (0.6.10-1) ... + Selecting previously unselected package ssh-import-id. + Preparing to unpack .../ssh-import-id_3.21-0ubuntu1_all.deb ... + Unpacking ssh-import-id (3.21-0ubuntu1) ... + Selecting previously unselected package supervisor. + Preparing to unpack .../supervisor_3.0b2-1_all.deb ... + Unpacking supervisor (3.0b2-1) ... + Processing triggers for ureadahead (0.100.0-16) ... + Processing triggers for mime-support (3.54ubuntu1.1) ... + Setting up libroken18-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libasn1-8-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libedit2:amd64 (3.1-20130712-2) ... + Setting up libkrb5support0:amd64 (1.12+dfsg-2ubuntu5.2) ... + Setting up libk5crypto3:amd64 (1.12+dfsg-2ubuntu5.2) ... + Setting up libkeyutils1:amd64 (1.5.6-1) ... + Setting up libkrb5-3:amd64 (1.12+dfsg-2ubuntu5.2) ... + Setting up libgssapi-krb5-2:amd64 (1.12+dfsg-2ubuntu5.2) ... + Setting up libhcrypto4-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libheimbase1-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libwind0-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libhx509-5-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libkrb5-26-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libheimntlm0-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libgssapi3-heimdal:amd64 (1.6~git20131207+dfsg-1ubuntu1.1) ... + Setting up libidn11:amd64 (1.28-1ubuntu2.1) ... + Setting up libsasl2-modules-db:amd64 (2.1.25.dfsg1-17build1) ... + Setting up libsasl2-2:amd64 (2.1.25.dfsg1-17build1) ... + Setting up libldap-2.4-2:amd64 (2.4.31-1+nmu2ubuntu8.3) ... + Setting up librtmp0:amd64 (2.4+20121230.gitdf6c518-1) ... + Setting up libxau6:amd64 (1:1.0.8-1) ... + Setting up libxdmcp6:amd64 (1:1.1.1-1) ... + Setting up libxcb1:amd64 (1.10-2ubuntu1) ... + Setting up libx11-data (2:1.6.2-1ubuntu2) ... + Setting up libx11-6:amd64 (2:1.6.2-1ubuntu2) ... + Setting up libxext6:amd64 (2:1.3.2-1ubuntu0.0.14.04.1) ... + Setting up libxmuu1:amd64 (2:1.1.1-1) ... + Setting up libck-connector0:amd64 (0.4.5-3.1ubuntu2) ... + Setting up libcurl3:amd64 (7.35.0-1ubuntu2.10) ... + Setting up libwrap0:amd64 (7.6.q-25) ... + Setting up libpython2.7-minimal:amd64 (2.7.6-8ubuntu0.2) ... + Setting up python2.7-minimal (2.7.6-8ubuntu0.2) ... + Setting up openssl (1.0.1f-1ubuntu2.21) ... + Setting up ca-certificates (20160104ubuntu0.14.04.1) ... + Setting up krb5-locales (1.12+dfsg-2ubuntu5.2) ... + Setting up libsasl2-modules:amd64 (2.1.25.dfsg1-17build1) ... + Setting up openssh-client (1:6.6p1-2ubuntu2.8) ... + Setting up wget (1.15-1ubuntu1.14.04.2) ... + Setting up xauth (1:1.0.7-1ubuntu1) ... + Setting up curl (7.35.0-1ubuntu2.10) ... + Setting up libpython2.7-stdlib:amd64 (2.7.6-8ubuntu0.2) ... + Setting up libpython-stdlib:amd64 (2.7.5-5ubuntu3) ... + Setting up ncurses-term (5.9+20140118-1ubuntu1) ... + Setting up openssh-sftp-server (1:6.6p1-2ubuntu2.8) ... + Setting up openssh-server (1:6.6p1-2ubuntu2.8) ... + Creating SSH2 RSA key; this may take some time ... + Creating SSH2 DSA key; this may take some time ... + Creating SSH2 ECDSA key; this may take some time ... + Creating SSH2 ED25519 key; this may take some time ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up python2.7 (2.7.6-8ubuntu0.2) ... + Setting up python-minimal (2.7.5-5ubuntu3) ... + Setting up python (2.7.5-5ubuntu3) ... + Setting up python-chardet (2.0.1-2build2) ... + Setting up python-pkg-resources (3.3-1ubuntu2) ... + Setting up python-six (1.5.2-1ubuntu1) ... + Setting up python-urllib3 (1.7.1-1ubuntu4) ... + Setting up python-requests (2.2.1-1ubuntu0.3) ... + Setting up tcpd (7.6.q-25) ... + Setting up python-meld3 (0.6.10-1) ... + Setting up ssh-import-id (3.21-0ubuntu1) ... + Setting up supervisor (3.0b2-1) ... + invoke-rc.d: policy-rc.d denied execution of start. + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + Processing triggers for ca-certificates (20160104ubuntu0.14.04.1) ... + Updating certificates in /etc/ssl/certs... 173 added, 0 removed; done. + Running hooks in /etc/ca-certificates/update.d....done. + Processing triggers for ureadahead (0.100.0-16) ... + Generating locales... + en_US.UTF-8... done + Generation complete. + gpg: no valid OpenPGP data found. + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 100 3149 100 3149 0 0 3063 0 0:00:01 0:00:01 --:--:-- 3066 + OK + Ign http://10.64.33.1:48080 binary/ InRelease + Get:1 http://10.64.33.1:48080 binary/ Release.gpg [819 B] + Get:2 http://10.64.33.1:48080 binary/ Release [1219 B] + Ign http://mirrors.aliyun.com trusty InRelease + Hit http://mirrors.aliyun.com trusty-updates InRelease + Hit http://mirrors.aliyun.com trusty-security InRelease + Get:3 http://10.64.33.1:48080 binary/ Packages [21.1 kB] + Hit http://mirrors.aliyun.com trusty Release.gpg + Hit http://mirrors.aliyun.com trusty-updates/main Sources + Hit http://mirrors.aliyun.com trusty-updates/restricted Sources + Hit http://mirrors.aliyun.com trusty-updates/universe Sources + Hit http://mirrors.aliyun.com trusty-updates/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty Release + Hit http://mirrors.aliyun.com trusty-security/main Sources + Hit http://mirrors.aliyun.com trusty-security/restricted Sources + Hit http://mirrors.aliyun.com trusty-security/universe Sources + Hit http://mirrors.aliyun.com trusty-security/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Fetched 23.1 kB in 4s (5673 B/s) + Reading package lists... + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + binutils build-essential clearwater-cluster-manager + clearwater-config-manager clearwater-diags-monitor clearwater-etcd + clearwater-monit clearwater-queue-manager cpp cpp-4.8 dnsmasq dnsmasq-base + dpkg-dev fakeroot g++ g++-4.8 gcc gcc-4.8 gnutls-bin inotify-tools + libalgorithm-diff-perl libalgorithm-diff-xs-perl libalgorithm-merge-perl + libasan0 libatomic1 libc-dev-bin libc6-dev libcloog-isl4 libdpkg-perl + libexpat1-dev libfakeroot libffi-dev libfile-fcntllock-perl libgcc-4.8-dev + libgmp10 libgomp1 libinotifytools0 libisl10 libitm1 libmnl0 libmpc3 libmpfr4 + libnetfilter-conntrack3 libnfnetlink0 libopts25 libpgm-5.1-0 libpython-dev + libpython2.7 libpython2.7-dev libquadmath0 libsensors4 libssl-dev libssl-doc + libstdc++-4.8-dev libtimedate-perl libtsan0 libzmq3 libzmq3-dev + linux-libc-dev make manpages manpages-dev ntp patch python-chardet-whl + python-colorama python-colorama-whl python-dev python-distlib + python-distlib-whl python-html5lib python-html5lib-whl python-pip + python-pip-whl python-requests-whl python-setuptools python-setuptools-whl + python-six-whl python-urllib3-whl python-virtualenv python-wheel + python2.7-dev python3-pkg-resources realpath sysstat xz-utils zlib1g-dev + Suggested packages: + binutils-doc clearwater-auto-config clearwater-auto-upgrade exim4 postfix + mail-transport-agent cpp-doc gcc-4.8-locales debian-keyring g++-multilib + g++-4.8-multilib gcc-4.8-doc libstdc++6-4.8-dbg gcc-multilib autoconf + automake1.9 libtool flex bison gdb gcc-doc gcc-4.8-multilib libgcc1-dbg + libgomp1-dbg libitm1-dbg libatomic1-dbg libasan0-dbg libtsan0-dbg + libquadmath0-dbg glibc-doc lm-sensors libstdc++-4.8-doc make-doc man-browser + ntp-doc apparmor ed diffutils-doc python-genshi python-lxml + python3-setuptools isag + Recommended packages: + python-dev-all + The following NEW packages will be installed: + binutils build-essential clearwater-auto-config-docker + clearwater-cluster-manager clearwater-config-manager + clearwater-diags-monitor clearwater-etcd clearwater-infrastructure + clearwater-management clearwater-monit clearwater-queue-manager cpp cpp-4.8 + dnsmasq dnsmasq-base dpkg-dev fakeroot g++ g++-4.8 gcc gcc-4.8 gnutls-bin + inotify-tools libalgorithm-diff-perl libalgorithm-diff-xs-perl + libalgorithm-merge-perl libasan0 libatomic1 libc-dev-bin libc6-dev + libcloog-isl4 libdpkg-perl libexpat1-dev libfakeroot libffi-dev + libfile-fcntllock-perl libgcc-4.8-dev libgmp10 libgomp1 libinotifytools0 + libisl10 libitm1 libmnl0 libmpc3 libmpfr4 libnetfilter-conntrack3 + libnfnetlink0 libopts25 libpgm-5.1-0 libpython-dev libpython2.7 + libpython2.7-dev libquadmath0 libsensors4 libssl-dev libssl-doc + libstdc++-4.8-dev libtimedate-perl libtsan0 libzmq3 libzmq3-dev + linux-libc-dev make manpages manpages-dev ntp patch python-chardet-whl + python-colorama python-colorama-whl python-dev python-distlib + python-distlib-whl python-html5lib python-html5lib-whl python-pip + python-pip-whl python-requests-whl python-setuptools python-setuptools-whl + python-six-whl python-urllib3-whl python-virtualenv python-wheel + python2.7-dev python3-pkg-resources realpath sysstat xz-utils zlib1g-dev + 0 upgraded, 90 newly installed, 0 to remove and 9 not upgraded. + Need to get 90.9 MB of archives. + After this operation, 236 MB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-auto-config-docker 1.0-161102.114310 [17.0 kB] + Get:2 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-infrastructure 1.0-161102.114310 [304 kB] + Get:3 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-monit 5.18-161028.000110 [271 kB] + Get:4 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-etcd 1.0-161108.170154 [7906 kB] + Get:5 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-cluster-manager 1.0-161108.170154 [3709 kB] + Get:6 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-queue-manager 1.0-161108.170154 [3690 kB] + Get:7 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-config-manager 1.0-161108.170154 [3648 kB] + Get:8 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-diags-monitor 1.0-161102.114310 [24.1 kB] + Get:9 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-management 1.0-161108.170154 [12.5 kB] + Get:10 http://mirrors.aliyun.com/ubuntu/ trusty/main libnfnetlink0 amd64 1.0.1-2 [15.0 kB] + Get:11 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libasan0 amd64 4.8.4-2ubuntu1~14.04.3 [63.1 kB] + Get:12 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libatomic1 amd64 4.8.4-2ubuntu1~14.04.3 [8636 B] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty/main libgmp10 amd64 2:5.1.3+dfsg-1ubuntu1 [218 kB] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty/main libisl10 amd64 0.12.2-1 [419 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty/main libcloog-isl4 amd64 0.18.2-1 [57.5 kB] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgomp1 amd64 4.8.4-2ubuntu1~14.04.3 [23.1 kB] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libitm1 amd64 4.8.4-2ubuntu1~14.04.3 [28.5 kB] + Get:18 http://mirrors.aliyun.com/ubuntu/ trusty/main libmpfr4 amd64 3.1.2-1 [203 kB] + Get:19 http://mirrors.aliyun.com/ubuntu/ trusty/main libopts25 amd64 1:5.18-2ubuntu2 [55.3 kB] + Get:20 http://mirrors.aliyun.com/ubuntu/ trusty/universe libpgm-5.1-0 amd64 5.1.118-1~dfsg-0.1ubuntu3 [156 kB] + Get:21 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpython2.7 amd64 2.7.6-8ubuntu0.2 [1039 kB] + Get:22 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libc-dev-bin amd64 2.19-0ubuntu6.9 [69.0 kB] + Get:23 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main linux-libc-dev amd64 3.13.0-101.148 [770 kB] + Get:24 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libc6-dev amd64 2.19-0ubuntu6.9 [1910 kB] + Get:25 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libexpat1-dev amd64 2.1.0-4ubuntu1.3 [115 kB] + Get:26 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpython2.7-dev amd64 2.7.6-8ubuntu0.2 [22.0 MB] + Get:27 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libquadmath0 amd64 4.8.4-2ubuntu1~14.04.3 [126 kB] + Get:28 http://mirrors.aliyun.com/ubuntu/ trusty/main libsensors4 amd64 1:3.3.4-2ubuntu1 [27.2 kB] + Get:29 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libtsan0 amd64 4.8.4-2ubuntu1~14.04.3 [94.9 kB] + Get:30 http://mirrors.aliyun.com/ubuntu/ trusty/universe libzmq3 amd64 4.0.4+dfsg-2 [129 kB] + Get:31 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main ntp amd64 1:4.2.6.p5+dfsg-3ubuntu2.14.04.10 [421 kB] + Get:32 http://mirrors.aliyun.com/ubuntu/ trusty/main libmnl0 amd64 1.0.3-3ubuntu1 [11.4 kB] + Get:33 http://mirrors.aliyun.com/ubuntu/ trusty/main libmpc3 amd64 1.0.1-1ubuntu1 [38.4 kB] + Get:34 http://mirrors.aliyun.com/ubuntu/ trusty/main libnetfilter-conntrack3 amd64 1.0.4-1 [45.9 kB] + Get:35 http://mirrors.aliyun.com/ubuntu/ trusty/main manpages all 3.54-1ubuntu1 [627 kB] + Get:36 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main binutils amd64 2.24-5ubuntu14.1 [2081 kB] + Get:37 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main cpp-4.8 amd64 4.8.4-2ubuntu1~14.04.3 [4595 kB] + Get:38 http://mirrors.aliyun.com/ubuntu/ trusty/main cpp amd64 4:4.8.2-1ubuntu6 [27.5 kB] + Get:39 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgcc-4.8-dev amd64 4.8.4-2ubuntu1~14.04.3 [1688 kB] + Get:40 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main gcc-4.8 amd64 4.8.4-2ubuntu1~14.04.3 [5047 kB] + Get:41 http://mirrors.aliyun.com/ubuntu/ trusty/main gcc amd64 4:4.8.2-1ubuntu6 [5098 B] + Get:42 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libstdc++-4.8-dev amd64 4.8.4-2ubuntu1~14.04.3 [1053 kB] + Get:43 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main g++-4.8 amd64 4.8.4-2ubuntu1~14.04.3 [18.1 MB] + Get:44 http://mirrors.aliyun.com/ubuntu/ trusty/main g++ amd64 4:4.8.2-1ubuntu6 [1490 B] + Get:45 http://mirrors.aliyun.com/ubuntu/ trusty/main make amd64 3.81-8.2ubuntu3 [119 kB] + Get:46 http://mirrors.aliyun.com/ubuntu/ trusty/main libtimedate-perl all 2.3000-1 [37.2 kB] + Get:47 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libdpkg-perl all 1.17.5ubuntu5.7 [179 kB] + Get:48 http://mirrors.aliyun.com/ubuntu/ trusty/main xz-utils amd64 5.1.1alpha+20120614-2ubuntu2 [78.8 kB] + Get:49 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main patch amd64 2.7.1-4ubuntu2.3 [86.4 kB] + Get:50 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main dpkg-dev all 1.17.5ubuntu5.7 [726 kB] + Get:51 http://mirrors.aliyun.com/ubuntu/ trusty/main build-essential amd64 11.6ubuntu6 [4838 B] + Get:52 http://mirrors.aliyun.com/ubuntu/ trusty/main libpython-dev amd64 2.7.5-5ubuntu3 [7078 B] + Get:53 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python2.7-dev amd64 2.7.6-8ubuntu0.2 [269 kB] + Get:54 http://mirrors.aliyun.com/ubuntu/ trusty/main python-dev amd64 2.7.5-5ubuntu3 [1166 B] + Get:55 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-colorama all 0.2.5-0.1ubuntu2 [18.4 kB] + Get:56 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-distlib all 0.1.8-1ubuntu1 [113 kB] + Get:57 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-html5lib all 0.999-3~ubuntu1 [83.5 kB] + Get:58 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python3-pkg-resources all 3.3-1ubuntu2 [31.7 kB] + Get:59 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-chardet-whl all 2.2.1-2~ubuntu1 [170 kB] + Get:60 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-colorama-whl all 0.2.5-0.1ubuntu2 [18.2 kB] + Get:61 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-distlib-whl all 0.1.8-1ubuntu1 [140 kB] + Get:62 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-html5lib-whl all 0.999-3~ubuntu1 [109 kB] + Get:63 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-six-whl all 1.5.2-1ubuntu1 [10.5 kB] + Get:64 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-urllib3-whl all 1.7.1-1ubuntu4 [64.0 kB] + Get:65 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-requests-whl all 2.2.1-1ubuntu0.3 [227 kB] + Get:66 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-setuptools-whl all 3.3-1ubuntu2 [244 kB] + Get:67 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-pip-whl all 1.5.4-1ubuntu4 [111 kB] + Get:68 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-setuptools all 3.3-1ubuntu2 [230 kB] + Get:69 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-pip all 1.5.4-1ubuntu4 [97.3 kB] + Get:70 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libffi-dev amd64 3.1~rc1+r3.0.13-12ubuntu0.1 [99.8 kB] + Get:71 http://mirrors.aliyun.com/ubuntu/ trusty/main zlib1g-dev amd64 1:1.2.8.dfsg-1ubuntu1 [183 kB] + Get:72 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libssl-dev amd64 1.0.1f-1ubuntu2.21 [1074 kB] + Get:73 http://mirrors.aliyun.com/ubuntu/ trusty/universe libzmq3-dev amd64 4.0.4+dfsg-2 [253 kB] + Get:74 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe python-virtualenv all 1.11.4-1ubuntu1 [1485 kB] + Get:75 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main dnsmasq-base amd64 2.68-1ubuntu0.1 [257 kB] + Get:76 http://mirrors.aliyun.com/ubuntu/ trusty-updates/universe dnsmasq all 2.68-1ubuntu0.1 [14.9 kB] + Get:77 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main gnutls-bin amd64 3.0.11+really2.12.23-12ubuntu2.5 [114 kB] + Get:78 http://mirrors.aliyun.com/ubuntu/ trusty/universe libinotifytools0 amd64 3.14-1ubuntu1 [17.0 kB] + Get:79 http://mirrors.aliyun.com/ubuntu/ trusty/universe inotify-tools amd64 3.14-1ubuntu1 [22.3 kB] + Get:80 http://mirrors.aliyun.com/ubuntu/ trusty/main realpath amd64 1.19 [14.5 kB] + Get:81 http://mirrors.aliyun.com/ubuntu/ trusty/main sysstat amd64 10.2.0-1 [283 kB] + Get:82 http://mirrors.aliyun.com/ubuntu/ trusty/main libfakeroot amd64 1.20-3ubuntu2 [25.4 kB] + Get:83 http://mirrors.aliyun.com/ubuntu/ trusty/main fakeroot amd64 1.20-3ubuntu2 [55.0 kB] + Get:84 http://mirrors.aliyun.com/ubuntu/ trusty/main libalgorithm-diff-perl all 1.19.02-3 [50.0 kB] + Get:85 http://mirrors.aliyun.com/ubuntu/ trusty/main libalgorithm-diff-xs-perl amd64 0.04-2build4 [12.6 kB] + Get:86 http://mirrors.aliyun.com/ubuntu/ trusty/main libalgorithm-merge-perl all 0.08-2 [12.7 kB] + Get:87 http://mirrors.aliyun.com/ubuntu/ trusty/main libfile-fcntllock-perl amd64 0.14-2build1 [15.9 kB] + Get:88 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libssl-doc all 1.0.1f-1ubuntu2.21 [971 kB] + Get:89 http://mirrors.aliyun.com/ubuntu/ trusty/main manpages-dev all 3.54-1ubuntu1 [1820 kB] + Get:90 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main python-wheel all 0.24.0-1~ubuntu1 [44.7 kB] + Extracting templates from packages: 100% + Preconfiguring packages ... + Fetched 90.9 MB in 1min 16s (1187 kB/s) + Selecting previously unselected package libnfnetlink0:amd64. + (Reading database ... 16382 files and directories currently installed.) + Preparing to unpack .../libnfnetlink0_1.0.1-2_amd64.deb ... + Unpacking libnfnetlink0:amd64 (1.0.1-2) ... + Selecting previously unselected package libasan0:amd64. + Preparing to unpack .../libasan0_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libasan0:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package libatomic1:amd64. + Preparing to unpack .../libatomic1_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libatomic1:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package libgmp10:amd64. + Preparing to unpack .../libgmp10_2%3a5.1.3+dfsg-1ubuntu1_amd64.deb ... + Unpacking libgmp10:amd64 (2:5.1.3+dfsg-1ubuntu1) ... + Selecting previously unselected package libisl10:amd64. + Preparing to unpack .../libisl10_0.12.2-1_amd64.deb ... + Unpacking libisl10:amd64 (0.12.2-1) ... + Selecting previously unselected package libcloog-isl4:amd64. + Preparing to unpack .../libcloog-isl4_0.18.2-1_amd64.deb ... + Unpacking libcloog-isl4:amd64 (0.18.2-1) ... + Selecting previously unselected package libgomp1:amd64. + Preparing to unpack .../libgomp1_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libgomp1:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package libitm1:amd64. + Preparing to unpack .../libitm1_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libitm1:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package libmpfr4:amd64. + Preparing to unpack .../libmpfr4_3.1.2-1_amd64.deb ... + Unpacking libmpfr4:amd64 (3.1.2-1) ... + Selecting previously unselected package libopts25:amd64. + Preparing to unpack .../libopts25_1%3a5.18-2ubuntu2_amd64.deb ... + Unpacking libopts25:amd64 (1:5.18-2ubuntu2) ... + Selecting previously unselected package libpgm-5.1-0:amd64. + Preparing to unpack .../libpgm-5.1-0_5.1.118-1~dfsg-0.1ubuntu3_amd64.deb ... + Unpacking libpgm-5.1-0:amd64 (5.1.118-1~dfsg-0.1ubuntu3) ... + Selecting previously unselected package libpython2.7:amd64. + Preparing to unpack .../libpython2.7_2.7.6-8ubuntu0.2_amd64.deb ... + Unpacking libpython2.7:amd64 (2.7.6-8ubuntu0.2) ... + Selecting previously unselected package libc-dev-bin. + Preparing to unpack .../libc-dev-bin_2.19-0ubuntu6.9_amd64.deb ... + Unpacking libc-dev-bin (2.19-0ubuntu6.9) ... + Selecting previously unselected package linux-libc-dev:amd64. + Preparing to unpack .../linux-libc-dev_3.13.0-101.148_amd64.deb ... + Unpacking linux-libc-dev:amd64 (3.13.0-101.148) ... + Selecting previously unselected package libc6-dev:amd64. + Preparing to unpack .../libc6-dev_2.19-0ubuntu6.9_amd64.deb ... + Unpacking libc6-dev:amd64 (2.19-0ubuntu6.9) ... + Selecting previously unselected package libexpat1-dev:amd64. + Preparing to unpack .../libexpat1-dev_2.1.0-4ubuntu1.3_amd64.deb ... + Unpacking libexpat1-dev:amd64 (2.1.0-4ubuntu1.3) ... + Selecting previously unselected package libpython2.7-dev:amd64. + Preparing to unpack .../libpython2.7-dev_2.7.6-8ubuntu0.2_amd64.deb ... + Unpacking libpython2.7-dev:amd64 (2.7.6-8ubuntu0.2) ... + Selecting previously unselected package libquadmath0:amd64. + Preparing to unpack .../libquadmath0_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libquadmath0:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package libsensors4:amd64. + Preparing to unpack .../libsensors4_1%3a3.3.4-2ubuntu1_amd64.deb ... + Unpacking libsensors4:amd64 (1:3.3.4-2ubuntu1) ... + Selecting previously unselected package libtsan0:amd64. + Preparing to unpack .../libtsan0_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libtsan0:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package libzmq3:amd64. + Preparing to unpack .../libzmq3_4.0.4+dfsg-2_amd64.deb ... + Unpacking libzmq3:amd64 (4.0.4+dfsg-2) ... + Selecting previously unselected package ntp. + Preparing to unpack .../ntp_1%3a4.2.6.p5+dfsg-3ubuntu2.14.04.10_amd64.deb ... + Unpacking ntp (1:4.2.6.p5+dfsg-3ubuntu2.14.04.10) ... + Selecting previously unselected package libmnl0:amd64. + Preparing to unpack .../libmnl0_1.0.3-3ubuntu1_amd64.deb ... + Unpacking libmnl0:amd64 (1.0.3-3ubuntu1) ... + Selecting previously unselected package libmpc3:amd64. + Preparing to unpack .../libmpc3_1.0.1-1ubuntu1_amd64.deb ... + Unpacking libmpc3:amd64 (1.0.1-1ubuntu1) ... + Selecting previously unselected package libnetfilter-conntrack3:amd64. + Preparing to unpack .../libnetfilter-conntrack3_1.0.4-1_amd64.deb ... + Unpacking libnetfilter-conntrack3:amd64 (1.0.4-1) ... + Selecting previously unselected package manpages. + Preparing to unpack .../manpages_3.54-1ubuntu1_all.deb ... + Unpacking manpages (3.54-1ubuntu1) ... + Selecting previously unselected package binutils. + Preparing to unpack .../binutils_2.24-5ubuntu14.1_amd64.deb ... + Unpacking binutils (2.24-5ubuntu14.1) ... + Selecting previously unselected package cpp-4.8. + Preparing to unpack .../cpp-4.8_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking cpp-4.8 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package cpp. + Preparing to unpack .../cpp_4%3a4.8.2-1ubuntu6_amd64.deb ... + Unpacking cpp (4:4.8.2-1ubuntu6) ... + Selecting previously unselected package libgcc-4.8-dev:amd64. + Preparing to unpack .../libgcc-4.8-dev_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libgcc-4.8-dev:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package gcc-4.8. + Preparing to unpack .../gcc-4.8_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking gcc-4.8 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package gcc. + Preparing to unpack .../gcc_4%3a4.8.2-1ubuntu6_amd64.deb ... + Unpacking gcc (4:4.8.2-1ubuntu6) ... + Selecting previously unselected package libstdc++-4.8-dev:amd64. + Preparing to unpack .../libstdc++-4.8-dev_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking libstdc++-4.8-dev:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package g++-4.8. + Preparing to unpack .../g++-4.8_4.8.4-2ubuntu1~14.04.3_amd64.deb ... + Unpacking g++-4.8 (4.8.4-2ubuntu1~14.04.3) ... + Selecting previously unselected package g++. + Preparing to unpack .../g++_4%3a4.8.2-1ubuntu6_amd64.deb ... + Unpacking g++ (4:4.8.2-1ubuntu6) ... + Selecting previously unselected package make. + Preparing to unpack .../make_3.81-8.2ubuntu3_amd64.deb ... + Unpacking make (3.81-8.2ubuntu3) ... + Selecting previously unselected package libtimedate-perl. + Preparing to unpack .../libtimedate-perl_2.3000-1_all.deb ... + Unpacking libtimedate-perl (2.3000-1) ... + Selecting previously unselected package libdpkg-perl. + Preparing to unpack .../libdpkg-perl_1.17.5ubuntu5.7_all.deb ... + Unpacking libdpkg-perl (1.17.5ubuntu5.7) ... + Selecting previously unselected package xz-utils. + Preparing to unpack .../xz-utils_5.1.1alpha+20120614-2ubuntu2_amd64.deb ... + Unpacking xz-utils (5.1.1alpha+20120614-2ubuntu2) ... + Selecting previously unselected package patch. + Preparing to unpack .../patch_2.7.1-4ubuntu2.3_amd64.deb ... + Unpacking patch (2.7.1-4ubuntu2.3) ... + Selecting previously unselected package dpkg-dev. + Preparing to unpack .../dpkg-dev_1.17.5ubuntu5.7_all.deb ... + Unpacking dpkg-dev (1.17.5ubuntu5.7) ... + Selecting previously unselected package build-essential. + Preparing to unpack .../build-essential_11.6ubuntu6_amd64.deb ... + Unpacking build-essential (11.6ubuntu6) ... + Selecting previously unselected package clearwater-auto-config-docker. + Preparing to unpack .../clearwater-auto-config-docker_1.0-161102.114310_all.deb ... + Unpacking clearwater-auto-config-docker (1.0-161102.114310) ... + Selecting previously unselected package libpython-dev:amd64. + Preparing to unpack .../libpython-dev_2.7.5-5ubuntu3_amd64.deb ... + Unpacking libpython-dev:amd64 (2.7.5-5ubuntu3) ... + Selecting previously unselected package python2.7-dev. + Preparing to unpack .../python2.7-dev_2.7.6-8ubuntu0.2_amd64.deb ... + Unpacking python2.7-dev (2.7.6-8ubuntu0.2) ... + Selecting previously unselected package python-dev. + Preparing to unpack .../python-dev_2.7.5-5ubuntu3_amd64.deb ... + Unpacking python-dev (2.7.5-5ubuntu3) ... + Selecting previously unselected package python-colorama. + Preparing to unpack .../python-colorama_0.2.5-0.1ubuntu2_all.deb ... + Unpacking python-colorama (0.2.5-0.1ubuntu2) ... + Selecting previously unselected package python-distlib. + Preparing to unpack .../python-distlib_0.1.8-1ubuntu1_all.deb ... + Unpacking python-distlib (0.1.8-1ubuntu1) ... + Selecting previously unselected package python-html5lib. + Preparing to unpack .../python-html5lib_0.999-3~ubuntu1_all.deb ... + Unpacking python-html5lib (0.999-3~ubuntu1) ... + Selecting previously unselected package python3-pkg-resources. + Preparing to unpack .../python3-pkg-resources_3.3-1ubuntu2_all.deb ... + Unpacking python3-pkg-resources (3.3-1ubuntu2) ... + Selecting previously unselected package python-chardet-whl. + Preparing to unpack .../python-chardet-whl_2.2.1-2~ubuntu1_all.deb ... + Unpacking python-chardet-whl (2.2.1-2~ubuntu1) ... + Selecting previously unselected package python-colorama-whl. + Preparing to unpack .../python-colorama-whl_0.2.5-0.1ubuntu2_all.deb ... + Unpacking python-colorama-whl (0.2.5-0.1ubuntu2) ... + Selecting previously unselected package python-distlib-whl. + Preparing to unpack .../python-distlib-whl_0.1.8-1ubuntu1_all.deb ... + Unpacking python-distlib-whl (0.1.8-1ubuntu1) ... + Selecting previously unselected package python-html5lib-whl. + Preparing to unpack .../python-html5lib-whl_0.999-3~ubuntu1_all.deb ... + Unpacking python-html5lib-whl (0.999-3~ubuntu1) ... + Selecting previously unselected package python-six-whl. + Preparing to unpack .../python-six-whl_1.5.2-1ubuntu1_all.deb ... + Unpacking python-six-whl (1.5.2-1ubuntu1) ... + Selecting previously unselected package python-urllib3-whl. + Preparing to unpack .../python-urllib3-whl_1.7.1-1ubuntu4_all.deb ... + Unpacking python-urllib3-whl (1.7.1-1ubuntu4) ... + Selecting previously unselected package python-requests-whl. + Preparing to unpack .../python-requests-whl_2.2.1-1ubuntu0.3_all.deb ... + Unpacking python-requests-whl (2.2.1-1ubuntu0.3) ... + Selecting previously unselected package python-setuptools-whl. + Preparing to unpack .../python-setuptools-whl_3.3-1ubuntu2_all.deb ... + Unpacking python-setuptools-whl (3.3-1ubuntu2) ... + Selecting previously unselected package python-pip-whl. + Preparing to unpack .../python-pip-whl_1.5.4-1ubuntu4_all.deb ... + Unpacking python-pip-whl (1.5.4-1ubuntu4) ... + Selecting previously unselected package python-setuptools. + Preparing to unpack .../python-setuptools_3.3-1ubuntu2_all.deb ... + Unpacking python-setuptools (3.3-1ubuntu2) ... + Selecting previously unselected package python-pip. + Preparing to unpack .../python-pip_1.5.4-1ubuntu4_all.deb ... + Unpacking python-pip (1.5.4-1ubuntu4) ... + Selecting previously unselected package libffi-dev:amd64. + Preparing to unpack .../libffi-dev_3.1~rc1+r3.0.13-12ubuntu0.1_amd64.deb ... + Unpacking libffi-dev:amd64 (3.1~rc1+r3.0.13-12ubuntu0.1) ... + Selecting previously unselected package zlib1g-dev:amd64. + Preparing to unpack .../zlib1g-dev_1%3a1.2.8.dfsg-1ubuntu1_amd64.deb ... + Unpacking zlib1g-dev:amd64 (1:1.2.8.dfsg-1ubuntu1) ... + Selecting previously unselected package libssl-dev:amd64. + Preparing to unpack .../libssl-dev_1.0.1f-1ubuntu2.21_amd64.deb ... + Unpacking libssl-dev:amd64 (1.0.1f-1ubuntu2.21) ... + Selecting previously unselected package libzmq3-dev:amd64. + Preparing to unpack .../libzmq3-dev_4.0.4+dfsg-2_amd64.deb ... + Unpacking libzmq3-dev:amd64 (4.0.4+dfsg-2) ... + Selecting previously unselected package python-virtualenv. + Preparing to unpack .../python-virtualenv_1.11.4-1ubuntu1_all.deb ... + Unpacking python-virtualenv (1.11.4-1ubuntu1) ... + Selecting previously unselected package dnsmasq-base. + Preparing to unpack .../dnsmasq-base_2.68-1ubuntu0.1_amd64.deb ... + Unpacking dnsmasq-base (2.68-1ubuntu0.1) ... + Selecting previously unselected package dnsmasq. + Preparing to unpack .../dnsmasq_2.68-1ubuntu0.1_all.deb ... + Unpacking dnsmasq (2.68-1ubuntu0.1) ... + Selecting previously unselected package gnutls-bin. + Preparing to unpack .../gnutls-bin_3.0.11+really2.12.23-12ubuntu2.5_amd64.deb ... + Unpacking gnutls-bin (3.0.11+really2.12.23-12ubuntu2.5) ... + Selecting previously unselected package clearwater-infrastructure. + Preparing to unpack .../clearwater-infrastructure_1.0-161102.114310_amd64.deb ... + Unpacking clearwater-infrastructure (1.0-161102.114310) ... + Selecting previously unselected package clearwater-monit. + Preparing to unpack .../clearwater-monit_5.18-161028.000110_amd64.deb ... + Unpacking clearwater-monit (5.18-161028.000110) ... + Selecting previously unselected package clearwater-etcd. + Preparing to unpack .../clearwater-etcd_1.0-161108.170154_all.deb ... + Unpacking clearwater-etcd (1.0-161108.170154) ... + Selecting previously unselected package clearwater-cluster-manager. + Preparing to unpack .../clearwater-cluster-manager_1.0-161108.170154_all.deb ... + Unpacking clearwater-cluster-manager (1.0-161108.170154) ... + Selecting previously unselected package clearwater-queue-manager. + Preparing to unpack .../clearwater-queue-manager_1.0-161108.170154_all.deb ... + Unpacking clearwater-queue-manager (1.0-161108.170154) ... + Selecting previously unselected package clearwater-config-manager. + Preparing to unpack .../clearwater-config-manager_1.0-161108.170154_all.deb ... + Unpacking clearwater-config-manager (1.0-161108.170154) ... + Selecting previously unselected package libinotifytools0. + Preparing to unpack .../libinotifytools0_3.14-1ubuntu1_amd64.deb ... + Unpacking libinotifytools0 (3.14-1ubuntu1) ... + Selecting previously unselected package inotify-tools. + Preparing to unpack .../inotify-tools_3.14-1ubuntu1_amd64.deb ... + Unpacking inotify-tools (3.14-1ubuntu1) ... + Selecting previously unselected package realpath. + Preparing to unpack .../realpath_1.19_amd64.deb ... + Unpacking realpath (1.19) ... + Selecting previously unselected package sysstat. + Preparing to unpack .../sysstat_10.2.0-1_amd64.deb ... + Unpacking sysstat (10.2.0-1) ... + Selecting previously unselected package clearwater-diags-monitor. + Preparing to unpack .../clearwater-diags-monitor_1.0-161102.114310_all.deb ... + Unpacking clearwater-diags-monitor (1.0-161102.114310) ... + Selecting previously unselected package clearwater-management. + Preparing to unpack .../clearwater-management_1.0-161108.170154_all.deb ... + Unpacking clearwater-management (1.0-161108.170154) ... + Selecting previously unselected package libfakeroot:amd64. + Preparing to unpack .../libfakeroot_1.20-3ubuntu2_amd64.deb ... + Unpacking libfakeroot:amd64 (1.20-3ubuntu2) ... + Selecting previously unselected package fakeroot. + Preparing to unpack .../fakeroot_1.20-3ubuntu2_amd64.deb ... + Unpacking fakeroot (1.20-3ubuntu2) ... + Selecting previously unselected package libalgorithm-diff-perl. + Preparing to unpack .../libalgorithm-diff-perl_1.19.02-3_all.deb ... + Unpacking libalgorithm-diff-perl (1.19.02-3) ... + Selecting previously unselected package libalgorithm-diff-xs-perl. + Preparing to unpack .../libalgorithm-diff-xs-perl_0.04-2build4_amd64.deb ... + Unpacking libalgorithm-diff-xs-perl (0.04-2build4) ... + Selecting previously unselected package libalgorithm-merge-perl. + Preparing to unpack .../libalgorithm-merge-perl_0.08-2_all.deb ... + Unpacking libalgorithm-merge-perl (0.08-2) ... + Selecting previously unselected package libfile-fcntllock-perl. + Preparing to unpack .../libfile-fcntllock-perl_0.14-2build1_amd64.deb ... + Unpacking libfile-fcntllock-perl (0.14-2build1) ... + Selecting previously unselected package libssl-doc. + Preparing to unpack .../libssl-doc_1.0.1f-1ubuntu2.21_all.deb ... + Unpacking libssl-doc (1.0.1f-1ubuntu2.21) ... + Selecting previously unselected package manpages-dev. + Preparing to unpack .../manpages-dev_3.54-1ubuntu1_all.deb ... + Unpacking manpages-dev (3.54-1ubuntu1) ... + Selecting previously unselected package python-wheel. + Preparing to unpack .../python-wheel_0.24.0-1~ubuntu1_all.deb ... + Unpacking python-wheel (0.24.0-1~ubuntu1) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libnfnetlink0:amd64 (1.0.1-2) ... + Setting up libasan0:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up libatomic1:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up libgmp10:amd64 (2:5.1.3+dfsg-1ubuntu1) ... + Setting up libisl10:amd64 (0.12.2-1) ... + Setting up libcloog-isl4:amd64 (0.18.2-1) ... + Setting up libgomp1:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up libitm1:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up libmpfr4:amd64 (3.1.2-1) ... + Setting up libopts25:amd64 (1:5.18-2ubuntu2) ... + Setting up libpgm-5.1-0:amd64 (5.1.118-1~dfsg-0.1ubuntu3) ... + Setting up libpython2.7:amd64 (2.7.6-8ubuntu0.2) ... + Setting up libc-dev-bin (2.19-0ubuntu6.9) ... + Setting up linux-libc-dev:amd64 (3.13.0-101.148) ... + Setting up libc6-dev:amd64 (2.19-0ubuntu6.9) ... + Setting up libexpat1-dev:amd64 (2.1.0-4ubuntu1.3) ... + Setting up libpython2.7-dev:amd64 (2.7.6-8ubuntu0.2) ... + Setting up libquadmath0:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up libsensors4:amd64 (1:3.3.4-2ubuntu1) ... + Setting up libtsan0:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up libzmq3:amd64 (4.0.4+dfsg-2) ... + Setting up ntp (1:4.2.6.p5+dfsg-3ubuntu2.14.04.10) ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up libmnl0:amd64 (1.0.3-3ubuntu1) ... + Setting up libmpc3:amd64 (1.0.1-1ubuntu1) ... + Setting up libnetfilter-conntrack3:amd64 (1.0.4-1) ... + Setting up manpages (3.54-1ubuntu1) ... + Setting up binutils (2.24-5ubuntu14.1) ... + Setting up cpp-4.8 (4.8.4-2ubuntu1~14.04.3) ... + Setting up cpp (4:4.8.2-1ubuntu6) ... + Setting up libgcc-4.8-dev:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up gcc-4.8 (4.8.4-2ubuntu1~14.04.3) ... + Setting up gcc (4:4.8.2-1ubuntu6) ... + Setting up libstdc++-4.8-dev:amd64 (4.8.4-2ubuntu1~14.04.3) ... + Setting up g++-4.8 (4.8.4-2ubuntu1~14.04.3) ... + Setting up g++ (4:4.8.2-1ubuntu6) ... + update-alternatives: using /usr/bin/g++ to provide /usr/bin/c++ (c++) in auto mode + Setting up make (3.81-8.2ubuntu3) ... + Setting up libtimedate-perl (2.3000-1) ... + Setting up libdpkg-perl (1.17.5ubuntu5.7) ... + Setting up xz-utils (5.1.1alpha+20120614-2ubuntu2) ... + update-alternatives: using /usr/bin/xz to provide /usr/bin/lzma (lzma) in auto mode + Setting up patch (2.7.1-4ubuntu2.3) ... + Setting up dpkg-dev (1.17.5ubuntu5.7) ... + Setting up build-essential (11.6ubuntu6) ... + Setting up clearwater-auto-config-docker (1.0-161102.114310) ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up libpython-dev:amd64 (2.7.5-5ubuntu3) ... + Setting up python2.7-dev (2.7.6-8ubuntu0.2) ... + Setting up python-dev (2.7.5-5ubuntu3) ... + Setting up python-colorama (0.2.5-0.1ubuntu2) ... + Setting up python-distlib (0.1.8-1ubuntu1) ... + Setting up python-html5lib (0.999-3~ubuntu1) ... + Setting up python3-pkg-resources (3.3-1ubuntu2) ... + Setting up python-chardet-whl (2.2.1-2~ubuntu1) ... + Setting up python-colorama-whl (0.2.5-0.1ubuntu2) ... + Setting up python-distlib-whl (0.1.8-1ubuntu1) ... + Setting up python-html5lib-whl (0.999-3~ubuntu1) ... + Setting up python-six-whl (1.5.2-1ubuntu1) ... + Setting up python-urllib3-whl (1.7.1-1ubuntu4) ... + Setting up python-requests-whl (2.2.1-1ubuntu0.3) ... + Setting up python-setuptools-whl (3.3-1ubuntu2) ... + Setting up python-pip-whl (1.5.4-1ubuntu4) ... + Setting up python-setuptools (3.3-1ubuntu2) ... + Setting up python-pip (1.5.4-1ubuntu4) ... + Setting up libffi-dev:amd64 (3.1~rc1+r3.0.13-12ubuntu0.1) ... + Setting up zlib1g-dev:amd64 (1:1.2.8.dfsg-1ubuntu1) ... + Setting up libssl-dev:amd64 (1.0.1f-1ubuntu2.21) ... + Setting up libzmq3-dev:amd64 (4.0.4+dfsg-2) ... + Setting up python-virtualenv (1.11.4-1ubuntu1) ... + Setting up dnsmasq-base (2.68-1ubuntu0.1) ... + Setting up dnsmasq (2.68-1ubuntu0.1) ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up gnutls-bin (3.0.11+really2.12.23-12ubuntu2.5) ... + Setting up libinotifytools0 (3.14-1ubuntu1) ... + Setting up inotify-tools (3.14-1ubuntu1) ... + Setting up realpath (1.19) ... + Setting up sysstat (10.2.0-1) ... + + Creating config file /etc/default/sysstat with new version + update-alternatives: using /usr/bin/sar.sysstat to provide /usr/bin/sar (sar) in auto mode + Setting up libfakeroot:amd64 (1.20-3ubuntu2) ... + Setting up fakeroot (1.20-3ubuntu2) ... + update-alternatives: using /usr/bin/fakeroot-sysv to provide /usr/bin/fakeroot (fakeroot) in auto mode + Setting up libalgorithm-diff-perl (1.19.02-3) ... + Setting up libalgorithm-diff-xs-perl (0.04-2build4) ... + Setting up libalgorithm-merge-perl (0.08-2) ... + Setting up libfile-fcntllock-perl (0.14-2build1) ... + Setting up libssl-doc (1.0.1f-1ubuntu2.21) ... + Setting up manpages-dev (3.54-1ubuntu1) ... + Setting up python-wheel (0.24.0-1~ubuntu1) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up clearwater-infrastructure (1.0-161102.114310) ... + Processing pyzmq-14.3.1-py2.7-linux-x86_64.egg + Copying pyzmq-14.3.1-py2.7-linux-x86_64.egg to /usr/local/lib/python2.7/dist-packages + Adding pyzmq 14.3.1 to easy-install.pth file + + Installed /usr/local/lib/python2.7/dist-packages/pyzmq-14.3.1-py2.7-linux-x86_64.egg + Processing dependencies for pyzmq==14.3.1 + Finished processing dependencies for pyzmq==14.3.1 + invoke-rc.d: policy-rc.d denied execution of start. + Processing triggers for ureadahead (0.100.0-16) ... + Setting up clearwater-monit (5.18-161028.000110) ... + invoke-rc.d: unknown initscript, /etc/init.d/clearwater-monit not found. + invoke-rc.d: policy-rc.d denied execution of start. + Processing triggers for ureadahead (0.100.0-16) ... + Setting up clearwater-diags-monitor (1.0-161102.114310) ... + Setting up clearwater-etcd (1.0-161108.170154) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up clearwater-cluster-manager (1.0-161108.170154) ... + Already using interpreter /usr/bin/python + New python executable in /usr/share/clearwater/clearwater-cluster-manager/env/bin/python + Installing setuptools, pip...done. + Processing clearwater_cluster_manager-1.0-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/clearwater_cluster_manager-1.0-py2.7.egg + Extracting clearwater_cluster_manager-1.0-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding clearwater-cluster-manager 1.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/clearwater_cluster_manager-1.0-py2.7.egg + Processing dependencies for clearwater-cluster-manager==1.0 + Searching for py2-ipaddress==3.4.1 + Best match: py2-ipaddress 3.4.1 + Processing py2_ipaddress-3.4.1-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/py2_ipaddress-3.4.1-py2.7.egg + Extracting py2_ipaddress-3.4.1-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding py2-ipaddress 3.4.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/py2_ipaddress-3.4.1-py2.7.egg + Searching for clearwater-etcd-shared + Best match: clearwater-etcd-shared 1.0 + Processing clearwater_etcd_shared-1.0-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/clearwater_etcd_shared-1.0-py2.7.egg + Extracting clearwater_etcd_shared-1.0-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding clearwater-etcd-shared 1.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/clearwater_etcd_shared-1.0-py2.7.egg + Searching for metaswitchcommon + Best match: metaswitchcommon 0.1 + Processing metaswitchcommon-0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Extracting metaswitchcommon-0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding metaswitchcommon 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Searching for prctl + Best match: prctl 1.0.1 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Extracting prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding prctl 1.0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Searching for pyyaml==3.11 + Best match: PyYAML 3.11 + Processing PyYAML-3.11-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/PyYAML-3.11-py2.7-linux-x86_64.egg + Extracting PyYAML-3.11-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding PyYAML 3.11 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/PyYAML-3.11-py2.7-linux-x86_64.egg + Searching for python-etcd==0.4.3 + Best match: python-etcd 0.4.3 + Processing python_etcd-0.4.3-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/python_etcd-0.4.3-py2.7.egg + Extracting python_etcd-0.4.3-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding python-etcd 0.4.3 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/python_etcd-0.4.3-py2.7.egg + Searching for urllib3==1.17 + Best match: urllib3 1.17 + Processing urllib3-1.17-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/urllib3-1.17-py2.7.egg + Extracting urllib3-1.17-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding urllib3 1.17 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/urllib3-1.17-py2.7.egg + Searching for pyzmq==15.2 + Best match: pyzmq 15.2.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Extracting pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + File "/usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/asyncio.py", line 79 + "{!r}".format(fileobj)) from None + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/tests/_test_asyncio.py", line 43 + yield from a.send(b'hi') + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/auth/asyncio.py", line 28 + events = yield from self.__poller.poll() + ^ + SyntaxError: invalid syntax + + Adding pyzmq 15.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Searching for docopt + Best match: docopt 0.6.2 + Processing docopt-0.6.2-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/docopt-0.6.2-py2.7.egg + Extracting docopt-0.6.2-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding docopt 0.6.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/docopt-0.6.2-py2.7.egg + Searching for monotonic==0.6 + Best match: monotonic 0.6 + Processing monotonic-0.6-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Extracting monotonic-0.6-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding monotonic 0.6 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Searching for cffi==1.5.2 + Best match: cffi 1.5.2 + Processing cffi-1.5.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Extracting cffi-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding cffi 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Searching for pycrypto==2.6.1 + Best match: pycrypto 2.6.1 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Extracting pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding pycrypto 2.6.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Searching for py-bcrypt + Best match: py-bcrypt 0.4 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Extracting py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding py-bcrypt 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Searching for dnspython + Best match: dnspython 1.15.0 + Processing dnspython-1.15.0-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/dnspython-1.15.0-py2.7.egg + Extracting dnspython-1.15.0-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding dnspython 1.15.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/dnspython-1.15.0-py2.7.egg + Searching for pycparser + Best match: pycparser 2.17 + Processing pycparser-2.17-py2.7.egg + creating /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Extracting pycparser-2.17-py2.7.egg to /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages + Adding pycparser 2.17 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-cluster-manager/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Finished processing dependencies for clearwater-cluster-manager==1.0 + Setting up clearwater-queue-manager (1.0-161108.170154) ... + Already using interpreter /usr/bin/python + New python executable in /usr/share/clearwater/clearwater-queue-manager/env/bin/python + Installing setuptools, pip...done. + Processing clearwater_queue_manager-1.0-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/clearwater_queue_manager-1.0-py2.7.egg + Extracting clearwater_queue_manager-1.0-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding clearwater-queue-manager 1.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/clearwater_queue_manager-1.0-py2.7.egg + Processing dependencies for clearwater-queue-manager==1.0 + Searching for clearwater-etcd-shared + Best match: clearwater-etcd-shared 1.0 + Processing clearwater_etcd_shared-1.0-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/clearwater_etcd_shared-1.0-py2.7.egg + Extracting clearwater_etcd_shared-1.0-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding clearwater-etcd-shared 1.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/clearwater_etcd_shared-1.0-py2.7.egg + Searching for metaswitchcommon + Best match: metaswitchcommon 0.1 + Processing metaswitchcommon-0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Extracting metaswitchcommon-0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding metaswitchcommon 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Searching for prctl + Best match: prctl 1.0.1 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Extracting prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding prctl 1.0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Searching for futures==3.0.5 + Best match: futures 3.0.5 + Processing futures-3.0.5-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/futures-3.0.5-py2.7.egg + Extracting futures-3.0.5-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding futures 3.0.5 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/futures-3.0.5-py2.7.egg + Searching for pyzmq==15.2 + Best match: pyzmq 15.2.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Extracting pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + File "/usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/asyncio.py", line 79 + "{!r}".format(fileobj)) from None + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/tests/_test_asyncio.py", line 43 + yield from a.send(b'hi') + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/auth/asyncio.py", line 28 + events = yield from self.__poller.poll() + ^ + SyntaxError: invalid syntax + + Adding pyzmq 15.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Searching for python-etcd==0.4.3 + Best match: python-etcd 0.4.3 + Processing python_etcd-0.4.3-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/python_etcd-0.4.3-py2.7.egg + Extracting python_etcd-0.4.3-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding python-etcd 0.4.3 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/python_etcd-0.4.3-py2.7.egg + Searching for urllib3==1.17 + Best match: urllib3 1.17 + Processing urllib3-1.17-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/urllib3-1.17-py2.7.egg + Extracting urllib3-1.17-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding urllib3 1.17 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/urllib3-1.17-py2.7.egg + Searching for docopt + Best match: docopt 0.6.2 + Processing docopt-0.6.2-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/docopt-0.6.2-py2.7.egg + Extracting docopt-0.6.2-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding docopt 0.6.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/docopt-0.6.2-py2.7.egg + Searching for pyyaml==3.11 + Best match: PyYAML 3.11 + Processing PyYAML-3.11-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/PyYAML-3.11-py2.7-linux-x86_64.egg + Extracting PyYAML-3.11-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding PyYAML 3.11 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/PyYAML-3.11-py2.7-linux-x86_64.egg + Searching for monotonic==0.6 + Best match: monotonic 0.6 + Processing monotonic-0.6-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Extracting monotonic-0.6-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding monotonic 0.6 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Searching for cffi==1.5.2 + Best match: cffi 1.5.2 + Processing cffi-1.5.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Extracting cffi-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding cffi 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Searching for pycrypto==2.6.1 + Best match: pycrypto 2.6.1 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Extracting pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding pycrypto 2.6.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Searching for py-bcrypt + Best match: py-bcrypt 0.4 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Extracting py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding py-bcrypt 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Searching for dnspython + Best match: dnspython 1.15.0 + Processing dnspython-1.15.0-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/dnspython-1.15.0-py2.7.egg + Extracting dnspython-1.15.0-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding dnspython 1.15.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/dnspython-1.15.0-py2.7.egg + Searching for pycparser + Best match: pycparser 2.17 + Processing pycparser-2.17-py2.7.egg + creating /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Extracting pycparser-2.17-py2.7.egg to /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages + Adding pycparser 2.17 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-queue-manager/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Finished processing dependencies for clearwater-queue-manager==1.0 + Processing triggers for ureadahead (0.100.0-16) ... + Setting up clearwater-config-manager (1.0-161108.170154) ... + Already using interpreter /usr/bin/python + New python executable in /usr/share/clearwater/clearwater-config-manager/env/bin/python + Installing setuptools, pip...done. + Processing clearwater_config_manager-1.0-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/clearwater_config_manager-1.0-py2.7.egg + Extracting clearwater_config_manager-1.0-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding clearwater-config-manager 1.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/clearwater_config_manager-1.0-py2.7.egg + Processing dependencies for clearwater-config-manager==1.0 + Searching for clearwater-etcd-shared + Best match: clearwater-etcd-shared 1.0 + Processing clearwater_etcd_shared-1.0-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/clearwater_etcd_shared-1.0-py2.7.egg + Extracting clearwater_etcd_shared-1.0-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding clearwater-etcd-shared 1.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/clearwater_etcd_shared-1.0-py2.7.egg + Searching for metaswitchcommon + Best match: metaswitchcommon 0.1 + Processing metaswitchcommon-0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Extracting metaswitchcommon-0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding metaswitchcommon 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Searching for prctl + Best match: prctl 1.0.1 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Extracting prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding prctl 1.0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Searching for pyyaml==3.11 + Best match: PyYAML 3.11 + Processing PyYAML-3.11-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/PyYAML-3.11-py2.7-linux-x86_64.egg + Extracting PyYAML-3.11-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding PyYAML 3.11 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/PyYAML-3.11-py2.7-linux-x86_64.egg + Searching for python-etcd==0.4.3 + Best match: python-etcd 0.4.3 + Processing python_etcd-0.4.3-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/python_etcd-0.4.3-py2.7.egg + Extracting python_etcd-0.4.3-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding python-etcd 0.4.3 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/python_etcd-0.4.3-py2.7.egg + Searching for urllib3==1.17 + Best match: urllib3 1.17 + Processing urllib3-1.17-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/urllib3-1.17-py2.7.egg + Extracting urllib3-1.17-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding urllib3 1.17 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/urllib3-1.17-py2.7.egg + Searching for pyzmq==15.2 + Best match: pyzmq 15.2.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Extracting pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + File "/usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/asyncio.py", line 79 + "{!r}".format(fileobj)) from None + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/tests/_test_asyncio.py", line 43 + yield from a.send(b'hi') + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/auth/asyncio.py", line 28 + events = yield from self.__poller.poll() + ^ + SyntaxError: invalid syntax + + Adding pyzmq 15.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Searching for docopt + Best match: docopt 0.6.2 + Processing docopt-0.6.2-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/docopt-0.6.2-py2.7.egg + Extracting docopt-0.6.2-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding docopt 0.6.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/docopt-0.6.2-py2.7.egg + Searching for monotonic==0.6 + Best match: monotonic 0.6 + Processing monotonic-0.6-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Extracting monotonic-0.6-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding monotonic 0.6 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Searching for cffi==1.5.2 + Best match: cffi 1.5.2 + Processing cffi-1.5.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Extracting cffi-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding cffi 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Searching for pycrypto==2.6.1 + Best match: pycrypto 2.6.1 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Extracting pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding pycrypto 2.6.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Searching for py-bcrypt + Best match: py-bcrypt 0.4 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Extracting py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding py-bcrypt 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Searching for dnspython + Best match: dnspython 1.15.0 + Processing dnspython-1.15.0-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/dnspython-1.15.0-py2.7.egg + Extracting dnspython-1.15.0-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding dnspython 1.15.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/dnspython-1.15.0-py2.7.egg + Searching for pycparser + Best match: pycparser 2.17 + Processing pycparser-2.17-py2.7.egg + creating /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Extracting pycparser-2.17-py2.7.egg to /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages + Adding pycparser 2.17 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-config-manager/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Finished processing dependencies for clearwater-config-manager==1.0 + Processing triggers for ureadahead (0.100.0-16) ... + Setting up clearwater-management (1.0-161108.170154) ... + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move '/tmp/hosts.5360' to '/etc/hosts': Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + hostname: you must be root to change the host name + mv: cannot move '/tmp/hosts.5378' to '/etc/hosts': Device or resource busy + Configuring monit for only localhost access + ...done. + ---> ca4a268c6af1 + Removing intermediate container 7b08f9c8fc13 + Step 7 : ENV LANG en_US.UTF-8 LANGUAGE en_US:en LC_ALL en_US.UTF-8 + ---> Running in eab63f3351d3 + ---> 150f445a88f1 + Removing intermediate container eab63f3351d3 + Step 8 : ONBUILD user root + ---> Running in ed3ab12df266 + ---> 5f7a3bf5502c + Removing intermediate container ed3ab12df266 + Step 9 : ONBUILD expose 22 + ---> Running in a7cbda1e83f8 + ---> 4c39f8841180 + Removing intermediate container a7cbda1e83f8 + Step 10 : ONBUILD cmd /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + ---> Running in 072a191bcfd3 + ---> dac4dc3a3f3e + Removing intermediate container 072a191bcfd3 + Successfully built dac4dc3a3f3e + [tangfx@localhost cw-docker]$ + + + [tangfx@localhost cw-docker]$ docker images | grep clearwater + docker.io/tangfeixiong/clearwater-base-onbuild 1611180614.gitrev-c2da9c9 dac4dc3a3f3e 20 seconds ago 540.6 MB + docker.io/tangfeixiong/clearwater-sprout 1611180614.gitrev-c2da9c9 f02626e19a4c 8 hours ago 631.9 MB + docker.io/tangfeixiong/clearwater-ralf 1611180614.gitrev-c2da9c9 a7181ba36557 8 hours ago 606.2 MB + docker.io/tangfeixiong/clearwater-homestead 1611180614.gitrev-c2da9c9 63ace52bb817 8 hours ago 759.9 MB + docker.io/tangfeixiong/clearwater-homer 1611180614.gitrev-c2da9c9 de4ca4fff31d 9 hours ago 690.4 MB + docker.io/tangfeixiong/clearwater-ellis 1611180614.gitrev-c2da9c9 b7b794efd5f8 9 hours ago 790.3 MB + docker.io/tangfeixiong/clearwater-bono 1611180614.gitrev-c2da9c9 168582028e03 9 hours ago 614 MB + tangfeixiong/clearwater-base-onbuild latest e3dad2f72092 31 hours ago 562.9 MB + tangfeixiong/clearwater-base latest e35fad0e4d71 31 hours ago 562.9 MB + clearwater/base latest 4a95e0ab2713 42 hours ago 570.4 MB diff --git a/hack/kubernetes-clearwater/clearwater/docs/bono-docker-build.md b/hack/kubernetes-clearwater/clearwater/docs/bono-docker-build.md new file mode 100755 index 0000000..cd45a68 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docs/bono-docker-build.md @@ -0,0 +1,227 @@ +Table of contents + +Customize `Dockerfile` to `docker build` + + [tangfx@localhost base-docker]$ mkdir ../bono-docker && cd ../bono-docker/ + + [tangfx@localhost bono-docker]$ vi Dockerfile + + [tangfx@localhost bono-docker]$ vi Makefile + + [tangfx@localhost bono-docker]$ ls + Dockerfile Makefile + + [tangfx@localhost bono-docker]$ make IMG_TAG=1611180614.gitrev-c2da9c9 + /usr/bin/docker build --tag=docker.io/tangfeixiong/clearwater-bono:1611180614.gitrev-c2da9c9 --file=../../../../bono/Dockerfile.tangfx ../../../../bono + Sending build context to Docker daemon 7.168 kB + Step 1 : FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 + # Executing 2 build triggers... + Step 1 : EXPOSE 22 + ---> Using cache + Step 1 : CMD /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + ---> Using cache + ---> e16796a8d558 + Step 2 : MAINTAINER tangfeixiong + ---> Using cache + ---> e8f24f33160d + Step 3 : USER root + ---> Using cache + ---> dbc83c55f6a4 + Step 4 : ADD *supervisord.conf /etc/supervisor/conf.d/ + ---> 68daa5b24d5c + Removing intermediate container b53f3c16b97f + Step 5 : RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes bono && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/bono && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes restund && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/restund && mv /etc/supervisor/conf.d/bono.supervisord.conf /etc/supervisor/conf.d/bono.conf && mv /etc/supervisor/conf.d/restund.supervisord.conf /etc/supervisor/conf.d/restund.conf && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf + ---> Running in bafe6000f788 + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Hit http://10.64.33.1:48080 binary/ Packages + Ign http://mirrors.aliyun.com trusty InRelease + Hit http://mirrors.aliyun.com trusty-updates InRelease + Hit http://mirrors.aliyun.com trusty-security InRelease + Hit http://mirrors.aliyun.com trusty Release.gpg + Hit http://mirrors.aliyun.com trusty Release + Hit http://mirrors.aliyun.com trusty-updates/main Sources + Hit http://mirrors.aliyun.com trusty-updates/restricted Sources + Hit http://mirrors.aliyun.com trusty-updates/universe Sources + Hit http://mirrors.aliyun.com trusty-updates/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/main Sources + Hit http://mirrors.aliyun.com trusty-security/restricted Sources + Hit http://mirrors.aliyun.com trusty-security/universe Sources + Hit http://mirrors.aliyun.com trusty-security/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Reading package lists... + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + clearwater-log-cleanup clearwater-socket-factory clearwater-tcp-scalability + libboost-filesystem1.54.0 libboost-regex1.54.0 libboost-system1.54.0 + libboost-thread1.54.0 libevent-2.0-5 libevent-core-2.0-5 + libevent-pthreads-2.0-5 libicu52 libpci3 libperl5.18 libsnmp-base libsnmp30 + sprout-libs + Suggested packages: + restund bono-dbg clearwater-logging snmp-mibs-downloader + The following NEW packages will be installed: + bono clearwater-log-cleanup clearwater-socket-factory + clearwater-tcp-scalability libboost-filesystem1.54.0 libboost-regex1.54.0 + libboost-system1.54.0 libboost-thread1.54.0 libevent-2.0-5 + libevent-core-2.0-5 libevent-pthreads-2.0-5 libicu52 libpci3 libperl5.18 + libsnmp-base libsnmp30 sprout-libs + 0 upgraded, 17 newly installed, 0 to remove and 7 not upgraded. + Need to get 11.5 MB of archives. + After this operation, 46.1 MB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp-base 5.7.2~dfsg-clearwater4 [224 kB] + Get:2 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp30 5.7.2~dfsg-clearwater4 [811 kB] + Get:3 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-tcp-scalability 1.0-161102.114310 [14.6 kB] + Get:4 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-log-cleanup 1.0-161102.114310 [13.3 kB] + Get:5 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout-libs 1.0-161104.171821 [1,551 kB] + Get:6 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-socket-factory 1.0-161102.114310 [32.0 kB] + Get:7 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ bono 1.0-161104.171821 [1,564 kB] + Get:8 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpci3 amd64 1:3.2.1-1ubuntu5.1 [26.3 kB] + Get:9 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-system1.54.0 amd64 1.54.0-4ubuntu3.1 [10.1 kB] + Get:10 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-filesystem1.54.0 amd64 1.54.0-4ubuntu3.1 [34.2 kB] + Get:11 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libicu52 amd64 52.1-3ubuntu0.4 [6,752 kB] + Get:12 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-regex1.54.0 amd64 1.54.0-4ubuntu3.1 [261 kB] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-thread1.54.0 amd64 1.54.0-4ubuntu3.1 [26.5 kB] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [126 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-core-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [78.3 kB] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-pthreads-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [6,126 B] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libperl5.18 amd64 5.18.2-2ubuntu1.1 [1,332 B] + Fetched 11.5 MB in 6s (1,672 kB/s) + Selecting previously unselected package libpci3:amd64. + (Reading database ... 24489 files and directories currently installed.) + Preparing to unpack .../libpci3_1%3a3.2.1-1ubuntu5.1_amd64.deb ... + Unpacking libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Selecting previously unselected package libboost-system1.54.0:amd64. + Preparing to unpack .../libboost-system1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libboost-filesystem1.54.0:amd64. + Preparing to unpack .../libboost-filesystem1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libicu52:amd64. + Preparing to unpack .../libicu52_52.1-3ubuntu0.4_amd64.deb ... + Unpacking libicu52:amd64 (52.1-3ubuntu0.4) ... + Selecting previously unselected package libboost-regex1.54.0:amd64. + Preparing to unpack .../libboost-regex1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libboost-thread1.54.0:amd64. + Preparing to unpack .../libboost-thread1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-thread1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libevent-2.0-5:amd64. + Preparing to unpack .../libevent-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-core-2.0-5:amd64. + Preparing to unpack .../libevent-core-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-pthreads-2.0-5:amd64. + Preparing to unpack .../libevent-pthreads-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libperl5.18. + Preparing to unpack .../libperl5.18_5.18.2-2ubuntu1.1_amd64.deb ... + Unpacking libperl5.18 (5.18.2-2ubuntu1.1) ... + Selecting previously unselected package libsnmp-base. + Preparing to unpack .../libsnmp-base_5.7.2~dfsg-clearwater4_all.deb ... + Unpacking libsnmp-base (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package libsnmp30:amd64. + Preparing to unpack .../libsnmp30_5.7.2~dfsg-clearwater4_amd64.deb ... + Unpacking libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package clearwater-tcp-scalability. + Preparing to unpack .../clearwater-tcp-scalability_1.0-161102.114310_all.deb ... + Unpacking clearwater-tcp-scalability (1.0-161102.114310) ... + Selecting previously unselected package clearwater-log-cleanup. + Preparing to unpack .../clearwater-log-cleanup_1.0-161102.114310_all.deb ... + Unpacking clearwater-log-cleanup (1.0-161102.114310) ... + Selecting previously unselected package sprout-libs. + Preparing to unpack .../sprout-libs_1.0-161104.171821_amd64.deb ... + Unpacking sprout-libs (1.0-161104.171821) ... + Selecting previously unselected package clearwater-socket-factory. + Preparing to unpack .../clearwater-socket-factory_1.0-161102.114310_amd64.deb ... + Unpacking clearwater-socket-factory (1.0-161102.114310) ... + Selecting previously unselected package bono. + Preparing to unpack .../bono_1.0-161104.171821_amd64.deb ... + Unpacking bono (1.0-161104.171821) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Setting up libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libicu52:amd64 (52.1-3ubuntu0.4) ... + Setting up libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libboost-thread1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libperl5.18 (5.18.2-2ubuntu1.1) ... + Setting up libsnmp-base (5.7.2~dfsg-clearwater4) ... + Setting up libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Setting up clearwater-tcp-scalability (1.0-161102.114310) ... + Setting up clearwater-log-cleanup (1.0-161102.114310) ... + Setting up sprout-libs (1.0-161104.171821) ... + Setting up clearwater-socket-factory (1.0-161102.114310) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up bono (1.0-161104.171821) ... + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + Processing triggers for ureadahead (0.100.0-16) ... + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Ign http://mirrors.aliyun.com trusty InRelease + Hit http://mirrors.aliyun.com trusty-updates InRelease + Hit http://mirrors.aliyun.com trusty-security InRelease + Hit http://10.64.33.1:48080 binary/ Packages + Hit http://mirrors.aliyun.com trusty Release.gpg + Hit http://mirrors.aliyun.com trusty-updates/main Sources + Hit http://mirrors.aliyun.com trusty-updates/restricted Sources + Hit http://mirrors.aliyun.com trusty-updates/universe Sources + Hit http://mirrors.aliyun.com trusty-updates/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty Release + Hit http://mirrors.aliyun.com trusty-security/main Sources + Hit http://mirrors.aliyun.com trusty-security/restricted Sources + Hit http://mirrors.aliyun.com trusty-security/universe Sources + Hit http://mirrors.aliyun.com trusty-security/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Reading package lists... + Reading package lists... + Building dependency tree... + Reading state information... + Suggested packages: + clearwater-logging clearwater-snmpd + The following NEW packages will be installed: + restund + 0 upgraded, 1 newly installed, 0 to remove and 7 not upgraded. + Need to get 610 kB of archives. + After this operation, 3,893 kB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ restund 1.0-161104.171821 [610 kB] + Fetched 610 kB in 0s (20.0 MB/s) + Selecting previously unselected package restund. + (Reading database ... 24743 files and directories currently installed.) + Preparing to unpack .../restund_1.0-161104.171821_amd64.deb ... + Unpacking restund (1.0-161104.171821) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up restund (1.0-161104.171821) ... + Processing triggers for ureadahead (0.100.0-16) ... + ---> 0d17f8365c52 + Removing intermediate container bafe6000f788 + Step 6 : EXPOSE 3478 3478/udp 5058 5060 5060/udp 5062 + ---> Running in d37874a8078d + ---> 168582028e03 + Removing intermediate container d37874a8078d + Successfully built 168582028e03 diff --git a/hack/kubernetes-clearwater/clearwater/docs/ellis-docker-build.md b/hack/kubernetes-clearwater/clearwater/docs/ellis-docker-build.md new file mode 100755 index 0000000..757fcd8 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docs/ellis-docker-build.md @@ -0,0 +1,661 @@ + +### Build docker image + +Customize `Dockerfile` to `docker build` + + [tangfx@localhost bono]$ mkdir -p ../ellis-docker && cd ../ellis-docker + + [tangfx@localhost ellis-docker]$ vi Dockerfile + + [tangfx@localhost ellis-docker]$ vi Makefile + + [tangfx@localhost ellis-docker]$ ls + Dockerfile Makefile + + [tangfx@localhost ellis-docker]$ make IMG_TAG=1611180614.gitrev-c2da9c9 + /usr/bin/docker build --tag=docker.io/tangfeixiong/clearwater-ellis:1611180614.gitrev-c2da9c9 --file=../../../../ellis/Dockerfile.tangfx ../../../../ellis + Sending build context to Docker daemon 8.704 kB + Step 1 : FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 + # Executing 2 build triggers... + Step 1 : EXPOSE 22 + ---> Using cache + Step 1 : CMD /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + ---> Using cache + ---> e16796a8d558 + Step 2 : MAINTAINER tangfeixiong + ---> Using cache + ---> e8f24f33160d + Step 3 : USER root + ---> Using cache + ---> dbc83c55f6a4 + Step 4 : ADD *supervisord.conf /etc/supervisor/conf.d/ + ---> 971f39810aaa + Removing intermediate container 227b22ee0ccd + Step 5 : RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes mysql-server && /etc/init.d/mysql start && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes ellis && /etc/init.d/mysql start && /usr/share/clearwater/ellis/env/bin/python /usr/share/clearwater/ellis/src/metaswitch/ellis/tools/create_numbers.py --start 6505550000 --count 1000 && mv /etc/supervisor/conf.d/ellis.supervisord.conf /etc/supervisor/conf.d/ellis.conf && mv /etc/supervisor/conf.d/mysql.supervisord.conf /etc/supervisor/conf.d/mysql.conf && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf + ---> Running in 5dda7c77353f + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Ign http://mirrors.aliyun.com trusty InRelease + Get:1 http://mirrors.aliyun.com trusty-updates InRelease [65.9 kB] + Get:2 http://mirrors.aliyun.com trusty-security InRelease [65.9 kB] + Hit http://10.64.33.1:48080 binary/ Packages + Hit http://mirrors.aliyun.com trusty Release.gpg + Get:3 http://mirrors.aliyun.com trusty-updates/main Sources [476 kB] + Get:4 http://mirrors.aliyun.com trusty-updates/restricted Sources [5,921 B] + Get:5 http://mirrors.aliyun.com trusty-updates/universe Sources [214 kB] + Get:6 http://mirrors.aliyun.com trusty-updates/main amd64 Packages [1,145 kB] + Get:7 http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages [20.4 kB] + Get:8 http://mirrors.aliyun.com trusty-updates/universe amd64 Packages [502 kB] + Hit http://mirrors.aliyun.com trusty Release + Get:9 http://mirrors.aliyun.com trusty-security/main Sources [153 kB] + Get:10 http://mirrors.aliyun.com trusty-security/restricted Sources [4,621 B] + Get:11 http://mirrors.aliyun.com trusty-security/universe Sources [54.0 kB] + Get:12 http://mirrors.aliyun.com trusty-security/main amd64 Packages [681 kB] + Get:13 http://mirrors.aliyun.com trusty-security/restricted amd64 Packages [17.0 kB] + Get:14 http://mirrors.aliyun.com trusty-security/universe amd64 Packages [188 kB] + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Fetched 3,590 kB in 5s (640 kB/s) + Reading package lists... + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-updates_universe_binary-amd64_Packages.gz is not what the server reported 501609 501619 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_source_Sources.gz is not what the server reported 153056 153059 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_binary-amd64_Packages.gz is not what the server reported 680542 680554 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_universe_binary-amd64_Packages.gz is not what the server reported 187712 187715 + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + libaio1 libdbd-mysql-perl libdbi-perl libhtml-template-perl libmysqlclient18 + libterm-readkey-perl mysql-client-5.5 mysql-client-core-5.5 mysql-common + mysql-server-5.5 mysql-server-core-5.5 psmisc + Suggested packages: + libclone-perl libmldbm-perl libnet-daemon-perl libplrpc-perl + libsql-statement-perl libipc-sharedcache-perl tinyca mailx + The following NEW packages will be installed: + libaio1 libdbd-mysql-perl libdbi-perl libhtml-template-perl libmysqlclient18 + libterm-readkey-perl mysql-client-5.5 mysql-client-core-5.5 mysql-common + mysql-server mysql-server-5.5 mysql-server-core-5.5 psmisc + 0 upgraded, 13 newly installed, 0 to remove and 9 not upgraded. + Need to get 9,310 kB of archives. + After this operation, 97.3 MB of additional disk space will be used. + Get:1 http://mirrors.aliyun.com/ubuntu/ trusty/main libaio1 amd64 0.3.109-4 [6,364 B] + Get:2 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main mysql-common all 5.5.53-0ubuntu0.14.04.1 [13.0 kB] + Get:3 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libmysqlclient18 amd64 5.5.53-0ubuntu0.14.04.1 [597 kB] + Get:4 http://mirrors.aliyun.com/ubuntu/ trusty/main libdbi-perl amd64 1.630-1 [879 kB] + Get:5 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libdbd-mysql-perl amd64 4.025-1ubuntu0.1 [87.6 kB] + Get:6 http://mirrors.aliyun.com/ubuntu/ trusty/main libterm-readkey-perl amd64 2.31-1 [27.4 kB] + Get:7 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main mysql-client-core-5.5 amd64 5.5.53-0ubuntu0.14.04.1 [708 kB] + Get:8 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main mysql-client-5.5 amd64 5.5.53-0ubuntu0.14.04.1 [1,587 kB] + Get:9 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main mysql-server-core-5.5 amd64 5.5.53-0ubuntu0.14.04.1 [3,284 kB] + Get:10 http://mirrors.aliyun.com/ubuntu/ trusty/main psmisc amd64 22.20-1ubuntu2 [53.2 kB] + Get:11 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main mysql-server-5.5 amd64 5.5.53-0ubuntu0.14.04.1 [1,990 kB] + Get:12 http://mirrors.aliyun.com/ubuntu/ trusty/main libhtml-template-perl all 2.95-1 [65.5 kB] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main mysql-server all 5.5.53-0ubuntu0.14.04.1 [11.3 kB] + Preconfiguring packages ... + Fetched 9,310 kB in 13s (668 kB/s) + Selecting previously unselected package libaio1:amd64. + (Reading database ... 24489 files and directories currently installed.) + Preparing to unpack .../libaio1_0.3.109-4_amd64.deb ... + Unpacking libaio1:amd64 (0.3.109-4) ... + Selecting previously unselected package mysql-common. + Preparing to unpack .../mysql-common_5.5.53-0ubuntu0.14.04.1_all.deb ... + Unpacking mysql-common (5.5.53-0ubuntu0.14.04.1) ... + Selecting previously unselected package libmysqlclient18:amd64. + Preparing to unpack .../libmysqlclient18_5.5.53-0ubuntu0.14.04.1_amd64.deb ... + Unpacking libmysqlclient18:amd64 (5.5.53-0ubuntu0.14.04.1) ... + Selecting previously unselected package libdbi-perl. + Preparing to unpack .../libdbi-perl_1.630-1_amd64.deb ... + Unpacking libdbi-perl (1.630-1) ... + Selecting previously unselected package libdbd-mysql-perl. + Preparing to unpack .../libdbd-mysql-perl_4.025-1ubuntu0.1_amd64.deb ... + Unpacking libdbd-mysql-perl (4.025-1ubuntu0.1) ... + Selecting previously unselected package libterm-readkey-perl. + Preparing to unpack .../libterm-readkey-perl_2.31-1_amd64.deb ... + Unpacking libterm-readkey-perl (2.31-1) ... + Selecting previously unselected package mysql-client-core-5.5. + Preparing to unpack .../mysql-client-core-5.5_5.5.53-0ubuntu0.14.04.1_amd64.deb ... + Unpacking mysql-client-core-5.5 (5.5.53-0ubuntu0.14.04.1) ... + Selecting previously unselected package mysql-client-5.5. + Preparing to unpack .../mysql-client-5.5_5.5.53-0ubuntu0.14.04.1_amd64.deb ... + Unpacking mysql-client-5.5 (5.5.53-0ubuntu0.14.04.1) ... + Selecting previously unselected package mysql-server-core-5.5. + Preparing to unpack .../mysql-server-core-5.5_5.5.53-0ubuntu0.14.04.1_amd64.deb ... + Unpacking mysql-server-core-5.5 (5.5.53-0ubuntu0.14.04.1) ... + Selecting previously unselected package psmisc. + Preparing to unpack .../psmisc_22.20-1ubuntu2_amd64.deb ... + Unpacking psmisc (22.20-1ubuntu2) ... + Setting up mysql-common (5.5.53-0ubuntu0.14.04.1) ... + Selecting previously unselected package mysql-server-5.5. + (Reading database ... 24872 files and directories currently installed.) + Preparing to unpack .../mysql-server-5.5_5.5.53-0ubuntu0.14.04.1_amd64.deb ... + Unpacking mysql-server-5.5 (5.5.53-0ubuntu0.14.04.1) ... + Selecting previously unselected package libhtml-template-perl. + Preparing to unpack .../libhtml-template-perl_2.95-1_all.deb ... + Unpacking libhtml-template-perl (2.95-1) ... + Selecting previously unselected package mysql-server. + Preparing to unpack .../mysql-server_5.5.53-0ubuntu0.14.04.1_all.deb ... + Unpacking mysql-server (5.5.53-0ubuntu0.14.04.1) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libaio1:amd64 (0.3.109-4) ... + Setting up libmysqlclient18:amd64 (5.5.53-0ubuntu0.14.04.1) ... + Setting up libdbi-perl (1.630-1) ... + Setting up libdbd-mysql-perl (4.025-1ubuntu0.1) ... + Setting up libterm-readkey-perl (2.31-1) ... + Setting up mysql-client-core-5.5 (5.5.53-0ubuntu0.14.04.1) ... + Setting up mysql-client-5.5 (5.5.53-0ubuntu0.14.04.1) ... + Setting up mysql-server-core-5.5 (5.5.53-0ubuntu0.14.04.1) ... + Setting up psmisc (22.20-1ubuntu2) ... + Setting up mysql-server-5.5 (5.5.53-0ubuntu0.14.04.1) ... + invoke-rc.d: policy-rc.d denied execution of stop. + invoke-rc.d: policy-rc.d denied execution of start. + Setting up libhtml-template-perl (2.95-1) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up mysql-server (5.5.53-0ubuntu0.14.04.1) ... + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + * Starting MySQL database server mysqld + ...done. + * Checking for tables which need an upgrade, are corrupt or were + not closed cleanly. + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Ign http://mirrors.aliyun.com trusty InRelease + Hit http://mirrors.aliyun.com trusty-updates InRelease + Hit http://mirrors.aliyun.com trusty-security InRelease + Hit http://10.64.33.1:48080 binary/ Packages + Hit http://mirrors.aliyun.com trusty Release.gpg + Hit http://mirrors.aliyun.com trusty-updates/main Sources + Hit http://mirrors.aliyun.com trusty-updates/restricted Sources + Hit http://mirrors.aliyun.com trusty-updates/universe Sources + Hit http://mirrors.aliyun.com trusty-updates/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-updates/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty Release + Hit http://mirrors.aliyun.com trusty-security/main Sources + Hit http://mirrors.aliyun.com trusty-security/restricted Sources + Hit http://mirrors.aliyun.com trusty-security/universe Sources + Hit http://mirrors.aliyun.com trusty-security/main amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty-security/universe amd64 Packages + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Reading package lists... + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + clearwater-log-cleanup clearwater-nginx fontconfig-config fonts-dejavu-core + geoip-database libfontconfig1 libfreetype6 libgd3 libgeoip1 libjbig0 + libjpeg-turbo8 libjpeg8 libtiff5 libvpx1 libxml2 libxpm4 libxslt1.1 nginx + nginx-common nginx-core sgml-base xml-core + Suggested packages: + clearwater-snmp-handler-alarm clearwater-logging clearwater-snmpd + libgd-tools geoip-bin fcgiwrap nginx-doc sgml-base-doc debhelper + The following NEW packages will be installed: + clearwater-log-cleanup clearwater-nginx ellis fontconfig-config + fonts-dejavu-core geoip-database libfontconfig1 libfreetype6 libgd3 + libgeoip1 libjbig0 libjpeg-turbo8 libjpeg8 libtiff5 libvpx1 libxml2 libxpm4 + libxslt1.1 nginx nginx-common nginx-core sgml-base xml-core + 0 upgraded, 23 newly installed, 0 to remove and 9 not upgraded. + Need to get 16.4 MB of archives. + After this operation, 30.8 MB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-log-cleanup 1.0-161102.114310 [13.3 kB] + Get:2 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-nginx 1.0-161028.000011 [19.7 kB] + Get:3 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ ellis 1.0-161104.121210 [11.5 MB] + Get:4 http://mirrors.aliyun.com/ubuntu/ trusty/main libgeoip1 amd64 1.6.0-1 [71.0 kB] + Get:5 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libxml2 amd64 2.9.1+dfsg1-3ubuntu4.8 [573 kB] + Get:6 http://mirrors.aliyun.com/ubuntu/ trusty/main sgml-base all 1.26+nmu4ubuntu1 [12.5 kB] + Get:7 http://mirrors.aliyun.com/ubuntu/ trusty/main fonts-dejavu-core all 2.34-1ubuntu1 [1,024 kB] + Get:8 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main fontconfig-config all 2.11.0-0ubuntu4.2 [47.4 kB] + Get:9 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfreetype6 amd64 2.5.2-1ubuntu2.5 [304 kB] + Get:10 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfontconfig1 amd64 2.11.0-0ubuntu4.2 [123 kB] + Get:11 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg-turbo8 amd64 1.3.0-0ubuntu2 [104 kB] + Get:12 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg8 amd64 8c-2ubuntu8 [2,194 B] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libjbig0 amd64 2.0-2ubuntu4.1 [26.1 kB] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libtiff5 amd64 4.0.3-7ubuntu0.4 [143 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty/main libvpx1 amd64 1.3.0-2 [556 kB] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty/main libxpm4 amd64 1:3.5.10-1 [38.3 kB] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgd3 amd64 2.1.0-3ubuntu0.5 [122 kB] + Get:18 http://mirrors.aliyun.com/ubuntu/ trusty/main libxslt1.1 amd64 1.1.28-2build1 [145 kB] + Get:19 http://mirrors.aliyun.com/ubuntu/ trusty/main geoip-database all 20140313-1 [1,196 kB] + Get:20 http://mirrors.aliyun.com/ubuntu/ trusty/main xml-core all 0.13+nmu2 [23.3 kB] + Get:21 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-common all 1.4.6-1ubuntu3.7 [19.0 kB] + Get:22 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-core amd64 1.4.6-1ubuntu3.7 [325 kB] + Get:23 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx all 1.4.6-1ubuntu3.7 [5,352 B] + Preconfiguring packages ... + Fetched 16.4 MB in 4s (3,948 kB/s) + Selecting previously unselected package libgeoip1:amd64. + (Reading database ... 24973 files and directories currently installed.) + Preparing to unpack .../libgeoip1_1.6.0-1_amd64.deb ... + Unpacking libgeoip1:amd64 (1.6.0-1) ... + Selecting previously unselected package libxml2:amd64. + Preparing to unpack .../libxml2_2.9.1+dfsg1-3ubuntu4.8_amd64.deb ... + Unpacking libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Selecting previously unselected package sgml-base. + Preparing to unpack .../sgml-base_1.26+nmu4ubuntu1_all.deb ... + Unpacking sgml-base (1.26+nmu4ubuntu1) ... + Selecting previously unselected package fonts-dejavu-core. + Preparing to unpack .../fonts-dejavu-core_2.34-1ubuntu1_all.deb ... + Unpacking fonts-dejavu-core (2.34-1ubuntu1) ... + Selecting previously unselected package fontconfig-config. + Preparing to unpack .../fontconfig-config_2.11.0-0ubuntu4.2_all.deb ... + Unpacking fontconfig-config (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libfreetype6:amd64. + Preparing to unpack .../libfreetype6_2.5.2-1ubuntu2.5_amd64.deb ... + Unpacking libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Selecting previously unselected package libfontconfig1:amd64. + Preparing to unpack .../libfontconfig1_2.11.0-0ubuntu4.2_amd64.deb ... + Unpacking libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libjpeg-turbo8:amd64. + Preparing to unpack .../libjpeg-turbo8_1.3.0-0ubuntu2_amd64.deb ... + Unpacking libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Selecting previously unselected package libjpeg8:amd64. + Preparing to unpack .../libjpeg8_8c-2ubuntu8_amd64.deb ... + Unpacking libjpeg8:amd64 (8c-2ubuntu8) ... + Selecting previously unselected package libjbig0:amd64. + Preparing to unpack .../libjbig0_2.0-2ubuntu4.1_amd64.deb ... + Unpacking libjbig0:amd64 (2.0-2ubuntu4.1) ... + Selecting previously unselected package libtiff5:amd64. + Preparing to unpack .../libtiff5_4.0.3-7ubuntu0.4_amd64.deb ... + Unpacking libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Selecting previously unselected package libvpx1:amd64. + Preparing to unpack .../libvpx1_1.3.0-2_amd64.deb ... + Unpacking libvpx1:amd64 (1.3.0-2) ... + Selecting previously unselected package libxpm4:amd64. + Preparing to unpack .../libxpm4_1%3a3.5.10-1_amd64.deb ... + Unpacking libxpm4:amd64 (1:3.5.10-1) ... + Selecting previously unselected package libgd3:amd64. + Preparing to unpack .../libgd3_2.1.0-3ubuntu0.5_amd64.deb ... + Unpacking libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Selecting previously unselected package libxslt1.1:amd64. + Preparing to unpack .../libxslt1.1_1.1.28-2build1_amd64.deb ... + Unpacking libxslt1.1:amd64 (1.1.28-2build1) ... + Selecting previously unselected package geoip-database. + Preparing to unpack .../geoip-database_20140313-1_all.deb ... + Unpacking geoip-database (20140313-1) ... + Selecting previously unselected package xml-core. + Preparing to unpack .../xml-core_0.13+nmu2_all.deb ... + Unpacking xml-core (0.13+nmu2) ... + Selecting previously unselected package clearwater-log-cleanup. + Preparing to unpack .../clearwater-log-cleanup_1.0-161102.114310_all.deb ... + Unpacking clearwater-log-cleanup (1.0-161102.114310) ... + Selecting previously unselected package nginx-common. + Preparing to unpack .../nginx-common_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx-common (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx-core. + Preparing to unpack .../nginx-core_1.4.6-1ubuntu3.7_amd64.deb ... + Unpacking nginx-core (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx. + Preparing to unpack .../nginx_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package clearwater-nginx. + Preparing to unpack .../clearwater-nginx_1.0-161028.000011_all.deb ... + Unpacking clearwater-nginx (1.0-161028.000011) ... + Selecting previously unselected package ellis. + Preparing to unpack .../ellis_1.0-161104.121210_amd64.deb ... + Unpacking ellis (1.0-161104.121210) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libgeoip1:amd64 (1.6.0-1) ... + Setting up libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Setting up sgml-base (1.26+nmu4ubuntu1) ... + Setting up fonts-dejavu-core (2.34-1ubuntu1) ... + Setting up fontconfig-config (2.11.0-0ubuntu4.2) ... + Setting up libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Setting up libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Setting up libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Setting up libjpeg8:amd64 (8c-2ubuntu8) ... + Setting up libjbig0:amd64 (2.0-2ubuntu4.1) ... + Setting up libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Setting up libvpx1:amd64 (1.3.0-2) ... + Setting up libxpm4:amd64 (1:3.5.10-1) ... + Setting up libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Setting up libxslt1.1:amd64 (1.1.28-2build1) ... + Setting up geoip-database (20140313-1) ... + Setting up xml-core (0.13+nmu2) ... + Setting up clearwater-log-cleanup (1.0-161102.114310) ... + Setting up nginx-common (1.4.6-1ubuntu3.7) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up nginx-core (1.4.6-1ubuntu3.7) ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up nginx (1.4.6-1ubuntu3.7) ... + Setting up clearwater-nginx (1.0-161028.000011) ... + Testing nginx configuration... + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + Site ping has been enabled. Run /etc/init.d/nginx reload to apply the changes. + Generating a 2048 bit RSA private key + ............................+++ + ..................................................................................................................+++ + writing new private key to 'nginx.key' + ----- + Signature ok + subject=/C=UK/ST=Unknown state/L=Unknown locality/O=Unknown organization/OU=Unknown organizational unit/CN=unknown.example/emailAddress=Unknown + Getting Private key + Adding 'diversion of /etc/nginx/nginx.conf to /etc/nginx/nginx.conf.clearwater-orig by clearwater-nginx' + Setting up ellis (1.0-161104.121210) ... + usermod: no changes + Already using interpreter /usr/bin/python + New python executable in /usr/share/clearwater/ellis/env/bin/python + Installing setuptools, pip...done. + Processing MySQL_python-1.2.5-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/MySQL_python-1.2.5-py2.7-linux-x86_64.egg + Extracting MySQL_python-1.2.5-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding MySQL-python 1.2.5 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/MySQL_python-1.2.5-py2.7-linux-x86_64.egg + Processing dependencies for MySQL-python==1.2.5 + Finished processing dependencies for MySQL-python==1.2.5 + Processing SQLAlchemy-1.0.9-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/SQLAlchemy-1.0.9-py2.7-linux-x86_64.egg + Extracting SQLAlchemy-1.0.9-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding SQLAlchemy 1.0.9 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/SQLAlchemy-1.0.9-py2.7-linux-x86_64.egg + Processing dependencies for SQLAlchemy==1.0.9 + Finished processing dependencies for SQLAlchemy==1.0.9 + Processing cffi-1.5.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Extracting cffi-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding cffi 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Processing dependencies for cffi==1.5.2 + Searching for pycparser + Best match: pycparser 2.17 + Processing pycparser-2.17-py2.7.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Extracting pycparser-2.17-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding pycparser 2.17 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Finished processing dependencies for cffi==1.5.2 + Processing ellis-0.1-py2.7.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/ellis-0.1-py2.7.egg + Extracting ellis-0.1-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding ellis 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/ellis-0.1-py2.7.egg + Processing dependencies for ellis==0.1 + Searching for pycurl==7.43.0 + Best match: pycurl 7.43.0 + Processing pycurl-7.43.0-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycurl-7.43.0-py2.7-linux-x86_64.egg + Extracting pycurl-7.43.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding pycurl 7.43.0 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycurl-7.43.0-py2.7-linux-x86_64.egg + Searching for prctl==1.0.1 + Best match: prctl 1.0.1 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Extracting prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding prctl 1.0.1 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Searching for phonenumbers==7.1.1 + Best match: phonenumbers 7.1.1 + Processing phonenumbers-7.1.1-py2.7.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/phonenumbers-7.1.1-py2.7.egg + Extracting phonenumbers-7.1.1-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding phonenumbers 7.1.1 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/phonenumbers-7.1.1-py2.7.egg + Searching for msgpack-python==0.4.6 + Best match: msgpack-python 0.4.6 + Processing msgpack_python-0.4.6-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/msgpack_python-0.4.6-py2.7-linux-x86_64.egg + Extracting msgpack_python-0.4.6-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding msgpack-python 0.4.6 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/msgpack_python-0.4.6-py2.7-linux-x86_64.egg + Searching for tornado==2.3 + Best match: tornado 2.3 + Processing tornado-2.3-py2.7.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/tornado-2.3-py2.7.egg + Extracting tornado-2.3-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding tornado 2.3 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/tornado-2.3-py2.7.egg + Searching for py-bcrypt==0.4 + Best match: py-bcrypt 0.4 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Extracting py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding py-bcrypt 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Finished processing dependencies for ellis==0.1 + Processing funcsigs-1.0.2-py2.7.egg + Copying funcsigs-1.0.2-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding funcsigs 1.0.2 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/funcsigs-1.0.2-py2.7.egg + Processing dependencies for funcsigs==1.0.2 + Finished processing dependencies for funcsigs==1.0.2 + Processing metaswitchcommon-0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Extracting metaswitchcommon-0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding metaswitchcommon 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Processing dependencies for metaswitchcommon==0.1 + Searching for monotonic==0.6 + Best match: monotonic 0.6 + Processing monotonic-0.6-py2.7.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Extracting monotonic-0.6-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding monotonic 0.6 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Searching for pyzmq==15.2 + Best match: pyzmq 15.2.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Extracting pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + File "/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/asyncio.py", line 79 + "{!r}".format(fileobj)) from None + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/tests/_test_asyncio.py", line 43 + yield from a.send(b'hi') + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/auth/asyncio.py", line 28 + events = yield from self.__poller.poll() + ^ + SyntaxError: invalid syntax + + Adding pyzmq 15.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Searching for pycrypto==2.6.1 + Best match: pycrypto 2.6.1 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Extracting pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding pycrypto 2.6.1 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Finished processing dependencies for metaswitchcommon==0.1 + Processing mock-2.0.0-py2.7.egg + Copying mock-2.0.0-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding mock 2.0.0 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/mock-2.0.0-py2.7.egg + Processing dependencies for mock==2.0.0 + Searching for six>=1.9 + Best match: six 1.10.0 + Processing six-1.10.0-py2.7.egg + Copying six-1.10.0-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding six 1.10.0 to easy-install.pth file + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Searching for pbr>=0.11 + Best match: pbr 1.6.0 + Processing pbr-1.6.0-py2.7.egg + Copying pbr-1.6.0-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + Adding pbr 1.6.0 to easy-install.pth file + Installing pbr script to /usr/share/clearwater/ellis/env/bin + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pbr-1.6.0-py2.7.egg + Finished processing dependencies for mock==2.0.0 + Processing monotonic-0.6-py2.7.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Extracting monotonic-0.6-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + monotonic 0.6 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Processing dependencies for monotonic==0.6 + Finished processing dependencies for monotonic==0.6 + Processing msgpack_python-0.4.6-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/msgpack_python-0.4.6-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/msgpack_python-0.4.6-py2.7-linux-x86_64.egg + Extracting msgpack_python-0.4.6-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + msgpack-python 0.4.6 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/msgpack_python-0.4.6-py2.7-linux-x86_64.egg + Processing dependencies for msgpack-python==0.4.6 + Finished processing dependencies for msgpack-python==0.4.6 + Processing pbr-1.6.0-py2.7.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pbr-1.6.0-py2.7.egg' (and everything under it) + Copying pbr-1.6.0-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + pbr 1.6.0 is already the active version in easy-install.pth + Installing pbr script to /usr/share/clearwater/ellis/env/bin + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pbr-1.6.0-py2.7.egg + Processing dependencies for pbr==1.6.0 + Finished processing dependencies for pbr==1.6.0 + Processing phonenumbers-7.1.1-py2.7.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/phonenumbers-7.1.1-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/phonenumbers-7.1.1-py2.7.egg + Extracting phonenumbers-7.1.1-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + phonenumbers 7.1.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/phonenumbers-7.1.1-py2.7.egg + Processing dependencies for phonenumbers==7.1.1 + Finished processing dependencies for phonenumbers==7.1.1 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Extracting prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + prctl 1.0.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Processing dependencies for prctl==1.0.1 + Finished processing dependencies for prctl==1.0.1 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Extracting py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + py-bcrypt 0.4 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Processing dependencies for py-bcrypt==0.4 + Finished processing dependencies for py-bcrypt==0.4 + Processing pycparser-2.17-py2.7.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Extracting pycparser-2.17-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + pycparser 2.17 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Processing dependencies for pycparser==2.17 + Finished processing dependencies for pycparser==2.17 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Extracting pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + pycrypto 2.6.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Processing dependencies for pycrypto==2.6.1 + Finished processing dependencies for pycrypto==2.6.1 + Processing pycurl-7.43.0-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycurl-7.43.0-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycurl-7.43.0-py2.7-linux-x86_64.egg + Extracting pycurl-7.43.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + pycurl 7.43.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pycurl-7.43.0-py2.7-linux-x86_64.egg + Processing dependencies for pycurl==7.43.0 + Finished processing dependencies for pycurl==7.43.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Extracting pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + File "/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/asyncio.py", line 79 + "{!r}".format(fileobj)) from None + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/tests/_test_asyncio.py", line 43 + yield from a.send(b'hi') + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/auth/asyncio.py", line 28 + events = yield from self.__poller.poll() + ^ + SyntaxError: invalid syntax + + pyzmq 15.2.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Processing dependencies for pyzmq==15.2.0 + Finished processing dependencies for pyzmq==15.2.0 + Processing six-1.10.0-py2.7.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg' (and everything under it) + Copying six-1.10.0-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + six 1.10.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Processing dependencies for six==1.10.0 + Finished processing dependencies for six==1.10.0 + Processing tornado-2.3-py2.7.egg + removing '/usr/share/clearwater/ellis/env/lib/python2.7/site-packages/tornado-2.3-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/tornado-2.3-py2.7.egg + Extracting tornado-2.3-py2.7.egg to /usr/share/clearwater/ellis/env/lib/python2.7/site-packages + tornado 2.3 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/ellis/env/lib/python2.7/site-packages/tornado-2.3-py2.7.egg + Processing dependencies for tornado==2.3 + Finished processing dependencies for tornado==2.3 + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move 鈥▒/tmp/hosts.1501鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + nginx: [warn] conflicting server name "10.120.160.2" on [::]:80, ignored + nginx: [warn] conflicting server name "10.120.160.2" on [::]:80, ignored + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + hostname: you must be root to change the host name + mv: cannot move 鈥▒/tmp/hosts.1585鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + Configuring monit for only localhost access + ...done. + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + Processing triggers for sgml-base (1.26+nmu4ubuntu1) ... + Processing triggers for ureadahead (0.100.0-16) ... + * Starting MySQL database server mysqld + ...done. + Created 1000 numbers, 0 already present in database + ---> 817d2931fb2c + Removing intermediate container 5dda7c77353f + Step 6 : EXPOSE 80 + ---> Running in 51102525dca1 + ---> b7b794efd5f8 + Removing intermediate container 51102525dca1 + Successfully built b7b794efd5f8 diff --git a/hack/kubernetes-clearwater/clearwater/docs/homer-docker-build.md b/hack/kubernetes-clearwater/clearwater/docs/homer-docker-build.md new file mode 100755 index 0000000..a2f93b4 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docs/homer-docker-build.md @@ -0,0 +1,688 @@ + + +Content + + [tangfx@localhost ellis-docker]$ mkdir -p ../homer-docker && cd ../homer-docker + + [tangfx@localhost homer-docker]$ vi Dockerfile + + [tangfx@localhost homer-docker]$ vi Makefile + + [tangfx@localhost homer-docker]$ ls + Dockerfile Makefile + + [tangfx@localhost homer-docker]$ make IMG_TAG=1611180614.gitrev-c2da9c9 + /usr/bin/docker build --tag=docker.io/tangfeixiong/clearwater-homer:1611180614.gitrev-c2da9c9 --file=../../../../homer/Dockerfile.tangfx ../../../../homer + Sending build context to Docker daemon 6.656 kB + Step 1 : FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 + # Executing 2 build triggers... + Step 1 : EXPOSE 22 + ---> Using cache + Step 1 : CMD /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + ---> Using cache + ---> e16796a8d558 + Step 2 : MAINTAINER tangfeixiong + ---> Using cache + ---> e8f24f33160d + Step 3 : USER root + ---> Using cache + ---> dbc83c55f6a4 + Step 4 : ADD *supervisord.conf /etc/supervisor/conf.d/ + ---> 62d3971561cd + Removing intermediate container 3412040c1211 + Step 5 : RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes homer && mv /etc/supervisor/conf.d/homer.supervisord.conf /etc/supervisor/conf.d/homer.conf && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf + ---> Running in 3d38f8ebb124 + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Ign http://mirrors.aliyun.com trusty InRelease + Get:1 http://mirrors.aliyun.com trusty-updates InRelease [65.9 kB] + Get:2 http://mirrors.aliyun.com trusty-security InRelease [65.9 kB] + Hit http://10.64.33.1:48080 binary/ Packages + Hit http://mirrors.aliyun.com trusty Release.gpg + Get:3 http://mirrors.aliyun.com trusty-updates/main Sources [476 kB] + Get:4 http://mirrors.aliyun.com trusty-updates/restricted Sources [5,921 B] + Get:5 http://mirrors.aliyun.com trusty-updates/universe Sources [214 kB] + Get:6 http://mirrors.aliyun.com trusty-updates/main amd64 Packages [1,145 kB] + Get:7 http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages [20.4 kB] + Get:8 http://mirrors.aliyun.com trusty-updates/universe amd64 Packages [502 kB] + Hit http://mirrors.aliyun.com trusty Release + Get:9 http://mirrors.aliyun.com trusty-security/main Sources [153 kB] + Get:10 http://mirrors.aliyun.com trusty-security/restricted Sources [4,621 B] + Get:11 http://mirrors.aliyun.com trusty-security/universe Sources [54.0 kB] + Get:12 http://mirrors.aliyun.com trusty-security/main amd64 Packages [681 kB] + Get:13 http://mirrors.aliyun.com trusty-security/restricted amd64 Packages [17.0 kB] + Get:14 http://mirrors.aliyun.com trusty-security/universe amd64 Packages [188 kB] + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Fetched 3,590 kB in 6s (588 kB/s) + Reading package lists... + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-updates_universe_binary-amd64_Packages.gz is not what the server reported 501609 501619 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_source_Sources.gz is not what the server reported 153056 153059 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_binary-amd64_Packages.gz is not what the server reported 680542 680554 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_universe_binary-amd64_Packages.gz is not what the server reported 187712 187715 + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + clearwater-log-cleanup clearwater-nginx crest fontconfig-config + fonts-dejavu-core geoip-database libfontconfig1 libfreetype6 libgd3 + libgeoip1 libjbig0 libjpeg-turbo8 libjpeg8 libtiff5 libvpx1 libxml2 + libxml2-dev libxpm4 libxslt1-dev libxslt1.1 nginx nginx-common nginx-core + python-zmq sgml-base xml-core + Suggested packages: + clearwater-snmp-handler-alarm clearwater-logging clearwater-snmpd + clearwater-secure-connections libgd-tools geoip-bin pkg-config fcgiwrap + nginx-doc sgml-base-doc debhelper + The following NEW packages will be installed: + clearwater-log-cleanup clearwater-nginx crest fontconfig-config + fonts-dejavu-core geoip-database homer libfontconfig1 libfreetype6 libgd3 + libgeoip1 libjbig0 libjpeg-turbo8 libjpeg8 libtiff5 libvpx1 libxml2 + libxml2-dev libxpm4 libxslt1-dev libxslt1.1 nginx nginx-common nginx-core + python-zmq sgml-base xml-core + 0 upgraded, 27 newly installed, 0 to remove and 9 not upgraded. + Need to get 15.4 MB of archives. + After this operation, 65.9 MB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-log-cleanup 1.0-161102.114310 [13.3 kB] + Get:2 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-nginx 1.0-161028.000011 [19.7 kB] + Get:3 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ crest 1.0-161104.120822 [9,294 kB] + Get:4 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ homer 1.0-161104.120822 [34.5 kB] + Get:5 http://mirrors.aliyun.com/ubuntu/ trusty/main libgeoip1 amd64 1.6.0-1 [71.0 kB] + Get:6 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libxml2 amd64 2.9.1+dfsg1-3ubuntu4.8 [573 kB] + Get:7 http://mirrors.aliyun.com/ubuntu/ trusty/main sgml-base all 1.26+nmu4ubuntu1 [12.5 kB] + Get:8 http://mirrors.aliyun.com/ubuntu/ trusty/main fonts-dejavu-core all 2.34-1ubuntu1 [1,024 kB] + Get:9 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main fontconfig-config all 2.11.0-0ubuntu4.2 [47.4 kB] + Get:10 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfreetype6 amd64 2.5.2-1ubuntu2.5 [304 kB] + Get:11 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfontconfig1 amd64 2.11.0-0ubuntu4.2 [123 kB] + Get:12 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg-turbo8 amd64 1.3.0-0ubuntu2 [104 kB] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg8 amd64 8c-2ubuntu8 [2,194 B] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libjbig0 amd64 2.0-2ubuntu4.1 [26.1 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libtiff5 amd64 4.0.3-7ubuntu0.4 [143 kB] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty/main libvpx1 amd64 1.3.0-2 [556 kB] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty/main libxpm4 amd64 1:3.5.10-1 [38.3 kB] + Get:18 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgd3 amd64 2.1.0-3ubuntu0.5 [122 kB] + Get:19 http://mirrors.aliyun.com/ubuntu/ trusty/main libxslt1.1 amd64 1.1.28-2build1 [145 kB] + Get:20 http://mirrors.aliyun.com/ubuntu/ trusty/main geoip-database all 20140313-1 [1,196 kB] + Get:21 http://mirrors.aliyun.com/ubuntu/ trusty/main xml-core all 0.13+nmu2 [23.3 kB] + Get:22 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-common all 1.4.6-1ubuntu3.7 [19.0 kB] + Get:23 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-core amd64 1.4.6-1ubuntu3.7 [325 kB] + Get:24 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx all 1.4.6-1ubuntu3.7 [5,352 B] + Get:25 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libxml2-dev amd64 2.9.1+dfsg1-3ubuntu4.8 [631 kB] + Get:26 http://mirrors.aliyun.com/ubuntu/ trusty/main libxslt1-dev amd64 1.1.28-2build1 [407 kB] + Get:27 http://mirrors.aliyun.com/ubuntu/ trusty/universe python-zmq amd64 14.0.1-1build2 [175 kB] + Preconfiguring packages ... + Fetched 15.4 MB in 6s (2,560 kB/s) + Selecting previously unselected package libgeoip1:amd64. + (Reading database ... 24489 files and directories currently installed.) + Preparing to unpack .../libgeoip1_1.6.0-1_amd64.deb ... + Unpacking libgeoip1:amd64 (1.6.0-1) ... + Selecting previously unselected package libxml2:amd64. + Preparing to unpack .../libxml2_2.9.1+dfsg1-3ubuntu4.8_amd64.deb ... + Unpacking libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Selecting previously unselected package sgml-base. + Preparing to unpack .../sgml-base_1.26+nmu4ubuntu1_all.deb ... + Unpacking sgml-base (1.26+nmu4ubuntu1) ... + Selecting previously unselected package fonts-dejavu-core. + Preparing to unpack .../fonts-dejavu-core_2.34-1ubuntu1_all.deb ... + Unpacking fonts-dejavu-core (2.34-1ubuntu1) ... + Selecting previously unselected package fontconfig-config. + Preparing to unpack .../fontconfig-config_2.11.0-0ubuntu4.2_all.deb ... + Unpacking fontconfig-config (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libfreetype6:amd64. + Preparing to unpack .../libfreetype6_2.5.2-1ubuntu2.5_amd64.deb ... + Unpacking libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Selecting previously unselected package libfontconfig1:amd64. + Preparing to unpack .../libfontconfig1_2.11.0-0ubuntu4.2_amd64.deb ... + Unpacking libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libjpeg-turbo8:amd64. + Preparing to unpack .../libjpeg-turbo8_1.3.0-0ubuntu2_amd64.deb ... + Unpacking libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Selecting previously unselected package libjpeg8:amd64. + Preparing to unpack .../libjpeg8_8c-2ubuntu8_amd64.deb ... + Unpacking libjpeg8:amd64 (8c-2ubuntu8) ... + Selecting previously unselected package libjbig0:amd64. + Preparing to unpack .../libjbig0_2.0-2ubuntu4.1_amd64.deb ... + Unpacking libjbig0:amd64 (2.0-2ubuntu4.1) ... + Selecting previously unselected package libtiff5:amd64. + Preparing to unpack .../libtiff5_4.0.3-7ubuntu0.4_amd64.deb ... + Unpacking libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Selecting previously unselected package libvpx1:amd64. + Preparing to unpack .../libvpx1_1.3.0-2_amd64.deb ... + Unpacking libvpx1:amd64 (1.3.0-2) ... + Selecting previously unselected package libxpm4:amd64. + Preparing to unpack .../libxpm4_1%3a3.5.10-1_amd64.deb ... + Unpacking libxpm4:amd64 (1:3.5.10-1) ... + Selecting previously unselected package libgd3:amd64. + Preparing to unpack .../libgd3_2.1.0-3ubuntu0.5_amd64.deb ... + Unpacking libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Selecting previously unselected package libxslt1.1:amd64. + Preparing to unpack .../libxslt1.1_1.1.28-2build1_amd64.deb ... + Unpacking libxslt1.1:amd64 (1.1.28-2build1) ... + Selecting previously unselected package geoip-database. + Preparing to unpack .../geoip-database_20140313-1_all.deb ... + Unpacking geoip-database (20140313-1) ... + Selecting previously unselected package xml-core. + Preparing to unpack .../xml-core_0.13+nmu2_all.deb ... + Unpacking xml-core (0.13+nmu2) ... + Selecting previously unselected package clearwater-log-cleanup. + Preparing to unpack .../clearwater-log-cleanup_1.0-161102.114310_all.deb ... + Unpacking clearwater-log-cleanup (1.0-161102.114310) ... + Selecting previously unselected package nginx-common. + Preparing to unpack .../nginx-common_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx-common (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx-core. + Preparing to unpack .../nginx-core_1.4.6-1ubuntu3.7_amd64.deb ... + Unpacking nginx-core (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx. + Preparing to unpack .../nginx_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package clearwater-nginx. + Preparing to unpack .../clearwater-nginx_1.0-161028.000011_all.deb ... + Unpacking clearwater-nginx (1.0-161028.000011) ... + Selecting previously unselected package libxml2-dev:amd64. + Preparing to unpack .../libxml2-dev_2.9.1+dfsg1-3ubuntu4.8_amd64.deb ... + Unpacking libxml2-dev:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Selecting previously unselected package libxslt1-dev:amd64. + Preparing to unpack .../libxslt1-dev_1.1.28-2build1_amd64.deb ... + Unpacking libxslt1-dev:amd64 (1.1.28-2build1) ... + Selecting previously unselected package python-zmq. + Preparing to unpack .../python-zmq_14.0.1-1build2_amd64.deb ... + Unpacking python-zmq (14.0.1-1build2) ... + Selecting previously unselected package crest. + Preparing to unpack .../crest_1.0-161104.120822_amd64.deb ... + Unpacking crest (1.0-161104.120822) ... + Selecting previously unselected package homer. + Preparing to unpack .../homer_1.0-161104.120822_amd64.deb ... + Unpacking homer (1.0-161104.120822) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libgeoip1:amd64 (1.6.0-1) ... + Setting up libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Setting up sgml-base (1.26+nmu4ubuntu1) ... + Setting up fonts-dejavu-core (2.34-1ubuntu1) ... + Setting up fontconfig-config (2.11.0-0ubuntu4.2) ... + Setting up libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Setting up libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Setting up libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Setting up libjpeg8:amd64 (8c-2ubuntu8) ... + Setting up libjbig0:amd64 (2.0-2ubuntu4.1) ... + Setting up libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Setting up libvpx1:amd64 (1.3.0-2) ... + Setting up libxpm4:amd64 (1:3.5.10-1) ... + Setting up libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Setting up libxslt1.1:amd64 (1.1.28-2build1) ... + Setting up geoip-database (20140313-1) ... + Setting up xml-core (0.13+nmu2) ... + Setting up clearwater-log-cleanup (1.0-161102.114310) ... + Setting up nginx-common (1.4.6-1ubuntu3.7) ... + Setting up libxml2-dev:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Setting up libxslt1-dev:amd64 (1.1.28-2build1) ... + Setting up python-zmq (14.0.1-1build2) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up nginx-core (1.4.6-1ubuntu3.7) ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up nginx (1.4.6-1ubuntu3.7) ... + Setting up clearwater-nginx (1.0-161028.000011) ... + Testing nginx configuration... + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + Site ping has been enabled. Run /etc/init.d/nginx reload to apply the changes. + Generating a 2048 bit RSA private key + .......+++ + ....................................+++ + writing new private key to 'nginx.key' + ----- + Signature ok + subject=/C=UK/ST=Unknown state/L=Unknown locality/O=Unknown organization/OU=Unknown organizational unit/CN=unknown.example/emailAddress=Unknown + Getting Private key + Adding 'diversion of /etc/nginx/nginx.conf to /etc/nginx/nginx.conf.clearwater-orig by clearwater-nginx' + Setting up crest (1.0-161104.120822) ... + Already using interpreter /usr/bin/python + New python executable in /usr/share/clearwater/crest/env/bin/python + Installing setuptools, pip...done. + Processing Twisted-16.5.0-py2.7-linux-x86_64.egg + Copying Twisted-16.5.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding Twisted 16.5.0 to easy-install.pth file + Installing trial script to /usr/share/clearwater/crest/env/bin + Installing conch script to /usr/share/clearwater/crest/env/bin + Installing ckeygen script to /usr/share/clearwater/crest/env/bin + Installing twist script to /usr/share/clearwater/crest/env/bin + Installing pyhtmlizer script to /usr/share/clearwater/crest/env/bin + Installing mailmail script to /usr/share/clearwater/crest/env/bin + Installing tkconch script to /usr/share/clearwater/crest/env/bin + Installing twistd script to /usr/share/clearwater/crest/env/bin + Installing cftp script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/Twisted-16.5.0-py2.7-linux-x86_64.egg + Processing dependencies for Twisted==16.5.0 + Searching for incremental>=16.10.1 + Best match: incremental 16.10.1 + Processing incremental-16.10.1-py2.7.egg + Copying incremental-16.10.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding incremental 16.10.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/incremental-16.10.1-py2.7.egg + Searching for constantly>=15.1 + Best match: constantly 15.1.0 + Processing constantly-15.1.0-py2.7.egg + Copying constantly-15.1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding constantly 15.1.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/constantly-15.1.0-py2.7.egg + Searching for zope.interface>=3.6.0 + Best match: zope.interface 4.3.2 + Processing zope.interface-4.3.2-py2.7-linux-x86_64.egg + Copying zope.interface-4.3.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding zope.interface 4.3.2 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/zope.interface-4.3.2-py2.7-linux-x86_64.egg + Finished processing dependencies for Twisted==16.5.0 + Processing cffi-1.5.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Extracting cffi-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cffi 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Processing dependencies for cffi==1.5.2 + Searching for pycparser + Best match: pycparser 2.17 + Processing pycparser-2.17-py2.7.egg + Copying pycparser-2.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pycparser 2.17 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Finished processing dependencies for cffi==1.5.2 + Processing cffi-1.8.3-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.8.3-py2.7-linux-x86_64.egg + Extracting cffi-1.8.3-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Removing cffi 1.5.2 from easy-install.pth file + Adding cffi 1.8.3 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.8.3-py2.7-linux-x86_64.egg + Processing dependencies for cffi==1.8.3 + Finished processing dependencies for cffi==1.8.3 + Processing constantly-15.1.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/constantly-15.1.0-py2.7.egg + Copying constantly-15.1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + constantly 15.1.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/constantly-15.1.0-py2.7.egg + Processing dependencies for constantly==15.1.0 + Finished processing dependencies for constantly==15.1.0 + Processing cql-1.4.0-py2.7.egg + Copying cql-1.4.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cql 1.4.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cql-1.4.0-py2.7.egg + Processing dependencies for cql==1.4.0 + Searching for thrift + Best match: thrift 0.9.3 + Processing thrift-0.9.3-py2.7-linux-x86_64.egg + Copying thrift-0.9.3-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding thrift 0.9.3 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/thrift-0.9.3-py2.7-linux-x86_64.egg + Finished processing dependencies for cql==1.4.0 + Processing crest-0.1-py2.7.egg + creating /usr/share/clearwater/crest/env/lib/python2.7/site-packages/crest-0.1-py2.7.egg + Extracting crest-0.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding crest 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/crest-0.1-py2.7.egg + Processing dependencies for crest==0.1 + Searching for monotonic==0.6 + Best match: monotonic 0.6 + Processing monotonic-0.6-py2.7.egg + Copying monotonic-0.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding monotonic 0.6 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Searching for prctl + Best match: prctl 1.0.1 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + Copying prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding prctl 1.0.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Searching for pure-sasl + Best match: pure-sasl 0.3.0 + Processing pure_sasl-0.3.0-py2.7.egg + Copying pure_sasl-0.3.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pure-sasl 0.3.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pure_sasl-0.3.0-py2.7.egg + Searching for msgpack-python==0.4.7 + Best match: msgpack-python 0.4.7 + Processing msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Copying msgpack_python-0.4.7-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding msgpack-python 0.4.7 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Searching for lxml + Best match: lxml 3.6.4 + Processing lxml-3.6.4-py2.7-linux-x86_64.egg + Copying lxml-3.6.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding lxml 3.6.4 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/lxml-3.6.4-py2.7-linux-x86_64.egg + Searching for cyclone==1.0 + Best match: cyclone 1.0 + Processing cyclone-1.0-py2.7.egg + Copying cyclone-1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cyclone 1.0 to easy-install.pth file + Installing cyclone script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cyclone-1.0-py2.7.egg + Searching for py-bcrypt + Best match: py-bcrypt 0.4 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + Copying py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding py-bcrypt 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Searching for pyzmq==15.2 + Best match: pyzmq 15.2.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + Copying pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pyzmq 15.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Searching for pyopenssl + Best match: pyOpenSSL 16.2.0 + Processing pyOpenSSL-16.2.0-py2.7.egg + Copying pyOpenSSL-16.2.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pyOpenSSL 16.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyOpenSSL-16.2.0-py2.7.egg + Searching for six>=1.5.2 + Best match: six 1.10.0 + Processing six-1.10.0-py2.7.egg + Copying six-1.10.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding six 1.10.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Searching for cryptography>=1.3.4 + Best match: cryptography 1.5.2 + Processing cryptography-1.5.2-py2.7-linux-x86_64.egg + Copying cryptography-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cryptography 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cryptography-1.5.2-py2.7-linux-x86_64.egg + Searching for ipaddress + Best match: ipaddress 1.0.17 + Processing ipaddress-1.0.17-py2.7.egg + Copying ipaddress-1.0.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding ipaddress 1.0.17 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/ipaddress-1.0.17-py2.7.egg + Searching for enum34 + Best match: enum34 1.1.6 + Processing enum34-1.1.6-py2.7.egg + Copying enum34-1.1.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding enum34 1.1.6 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/enum34-1.1.6-py2.7.egg + Searching for setuptools>=11.3 + Best match: setuptools 24.0.0 + Processing setuptools-24.0.0-py2.7.egg + Copying setuptools-24.0.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding setuptools 24.0.0 to easy-install.pth file + Installing easy_install script to /usr/share/clearwater/crest/env/bin + Installing easy_install-2.7 script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/setuptools-24.0.0-py2.7.egg + Searching for pyasn1>=0.1.8 + Best match: pyasn1 0.1.9 + Processing pyasn1-0.1.9-py2.7.egg + + Note: Bypassing https://pypi.python.org/packages/source/c/certifi/certifi-2016.2.28.tar.gz#md5=5d672aa766e1f773c75cfeccd02d3650 (disallowed host; see http://bit.ly/1dg9ijs for details). + + + Note: Bypassing https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2 (disallowed host; see http://bit.ly/1dg9ijs for details). + + Copying pyasn1-0.1.9-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pyasn1 0.1.9 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyasn1-0.1.9-py2.7.egg + Searching for idna>=2.0 + Best match: idna 2.1 + Processing idna-2.1-py2.7.egg + Copying idna-2.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding idna 2.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/idna-2.1-py2.7.egg + Finished processing dependencies for crest==0.1 + Processing cryptography-1.5.2-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/cryptography-1.5.2-py2.7-linux-x86_64.egg' (and everything under it) + Copying cryptography-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + cryptography 1.5.2 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cryptography-1.5.2-py2.7-linux-x86_64.egg + Processing dependencies for cryptography==1.5.2 + Finished processing dependencies for cryptography==1.5.2 + Processing cyclone-1.0-py2.7.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/cyclone-1.0-py2.7.egg' (and everything under it) + Copying cyclone-1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + cyclone 1.0 is already the active version in easy-install.pth + Installing cyclone script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cyclone-1.0-py2.7.egg + Processing dependencies for cyclone==1.0 + Finished processing dependencies for cyclone==1.0 + Processing enum34-1.1.6-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/enum34-1.1.6-py2.7.egg + Copying enum34-1.1.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + enum34 1.1.6 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/enum34-1.1.6-py2.7.egg + Processing dependencies for enum34==1.1.6 + Finished processing dependencies for enum34==1.1.6 + Processing idna-2.1-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/idna-2.1-py2.7.egg + Copying idna-2.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + idna 2.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/idna-2.1-py2.7.egg + Processing dependencies for idna==2.1 + Finished processing dependencies for idna==2.1 + Processing incremental-16.10.1-py2.7.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/incremental-16.10.1-py2.7.egg' (and everything under it) + Copying incremental-16.10.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + incremental 16.10.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/incremental-16.10.1-py2.7.egg + Processing dependencies for incremental==16.10.1 + Finished processing dependencies for incremental==16.10.1 + Processing ipaddress-1.0.17-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/ipaddress-1.0.17-py2.7.egg + Copying ipaddress-1.0.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + ipaddress 1.0.17 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/ipaddress-1.0.17-py2.7.egg + Processing dependencies for ipaddress==1.0.17 + Finished processing dependencies for ipaddress==1.0.17 + Processing lxml-3.6.4-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/lxml-3.6.4-py2.7-linux-x86_64.egg' (and everything under it) + Copying lxml-3.6.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + lxml 3.6.4 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/lxml-3.6.4-py2.7-linux-x86_64.egg + Processing dependencies for lxml==3.6.4 + Finished processing dependencies for lxml==3.6.4 + Processing metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Copying metaswitchcommon-0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding metaswitchcommon 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Processing dependencies for metaswitchcommon==0.1 + Searching for pycrypto==2.6.1 + Best match: pycrypto 2.6.1 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + Copying pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pycrypto 2.6.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Finished processing dependencies for metaswitchcommon==0.1 + Processing monotonic-0.6-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Copying monotonic-0.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + monotonic 0.6 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Processing dependencies for monotonic==0.6 + Finished processing dependencies for monotonic==0.6 + Processing msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Copying msgpack_python-0.4.7-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + msgpack-python 0.4.7 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Processing dependencies for msgpack-python==0.4.7 + Finished processing dependencies for msgpack-python==0.4.7 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Copying prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + prctl 1.0.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Processing dependencies for prctl==1.0.1 + Finished processing dependencies for prctl==1.0.1 + Processing pure_sasl-0.3.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pure_sasl-0.3.0-py2.7.egg + Copying pure_sasl-0.3.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pure-sasl 0.3.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pure_sasl-0.3.0-py2.7.egg + Processing dependencies for pure-sasl==0.3.0 + Finished processing dependencies for pure-sasl==0.3.0 + Processing pyOpenSSL-16.2.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyOpenSSL-16.2.0-py2.7.egg + Copying pyOpenSSL-16.2.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pyOpenSSL 16.2.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyOpenSSL-16.2.0-py2.7.egg + Processing dependencies for pyOpenSSL==16.2.0 + Finished processing dependencies for pyOpenSSL==16.2.0 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Copying py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + py-bcrypt 0.4 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Processing dependencies for py-bcrypt==0.4 + Finished processing dependencies for py-bcrypt==0.4 + Processing pyasn1-0.1.9-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyasn1-0.1.9-py2.7.egg + Copying pyasn1-0.1.9-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pyasn1 0.1.9 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyasn1-0.1.9-py2.7.egg + Processing dependencies for pyasn1==0.1.9 + Finished processing dependencies for pyasn1==0.1.9 + Processing pycparser-2.17-py2.7.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg' (and everything under it) + Copying pycparser-2.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pycparser 2.17 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Processing dependencies for pycparser==2.17 + Finished processing dependencies for pycparser==2.17 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Copying pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pycrypto 2.6.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Processing dependencies for pycrypto==2.6.1 + Finished processing dependencies for pycrypto==2.6.1 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg' (and everything under it) + Copying pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pyzmq 15.2.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Processing dependencies for pyzmq==15.2.0 + Finished processing dependencies for pyzmq==15.2.0 + Processing setuptools-24.0.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/setuptools-24.0.0-py2.7.egg + Copying setuptools-24.0.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + setuptools 24.0.0 is already the active version in easy-install.pth + Installing easy_install script to /usr/share/clearwater/crest/env/bin + Installing easy_install-2.7 script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/setuptools-24.0.0-py2.7.egg + Processing dependencies for setuptools==24.0.0 + Finished processing dependencies for setuptools==24.0.0 + Processing six-1.10.0-py2.7.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg' (and everything under it) + + Note: Bypassing https://pypi.python.org/packages/source/c/certifi/certifi-2016.2.28.tar.gz#md5=5d672aa766e1f773c75cfeccd02d3650 (disallowed host; see http://bit.ly/1dg9ijs for details). + + + Note: Bypassing https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2 (disallowed host; see http://bit.ly/1dg9ijs for details). + + Copying six-1.10.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + six 1.10.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Processing dependencies for six==1.10.0 + Finished processing dependencies for six==1.10.0 + Processing telephus-1.0.0b1-py2.7.egg + Copying telephus-1.0.0b1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding telephus 1.0.0b1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/telephus-1.0.0b1-py2.7.egg + Processing dependencies for telephus==1.0.0b1 + Finished processing dependencies for telephus==1.0.0b1 + Processing thrift-0.9.3-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/thrift-0.9.3-py2.7-linux-x86_64.egg + Copying thrift-0.9.3-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + thrift 0.9.3 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/thrift-0.9.3-py2.7-linux-x86_64.egg + Processing dependencies for thrift==0.9.3 + Finished processing dependencies for thrift==0.9.3 + Processing zope.interface-4.3.2-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/zope.interface-4.3.2-py2.7-linux-x86_64.egg' (and everything under it) + Copying zope.interface-4.3.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + zope.interface 4.3.2 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/zope.interface-4.3.2-py2.7-linux-x86_64.egg + Processing dependencies for zope.interface==4.3.2 + Finished processing dependencies for zope.interface==4.3.2 + Setting up homer (1.0-161104.120822) ... + Creating /usr/share/clearwater/homer/python/packages/site.py + Processing homer-0.1-py2.7.egg + creating /usr/share/clearwater/homer/python/packages/homer-0.1-py2.7.egg + Extracting homer-0.1-py2.7.egg to /usr/share/clearwater/homer/python/packages + Adding homer 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/homer/python/packages/homer-0.1-py2.7.egg + Processing dependencies for homer==0.1 + Finished processing dependencies for homer==0.1 + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move 鈥▒/tmp/hosts.555鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + hostname: you must be root to change the host name + mv: cannot move 鈥▒/tmp/hosts.625鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + Configuring monit for only localhost access + ...done. + /var/lib/dpkg/info/homer.postinst: 69: /var/lib/dpkg/info/homer.postinst: /usr/share/clearwater/cassandra-schemas/homer.sh: not found + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + Processing triggers for sgml-base (1.26+nmu4ubuntu1) ... + Processing triggers for ureadahead (0.100.0-16) ... + ---> 89a783bb541a + Removing intermediate container 3d38f8ebb124 + Step 6 : EXPOSE 7888 + ---> Running in 8a5e2d29db3d + ---> de4ca4fff31d + Removing intermediate container 8a5e2d29db3d + Successfully built de4ca4fff31d diff --git a/hack/kubernetes-clearwater/clearwater/docs/homestead-docker-build.md b/hack/kubernetes-clearwater/clearwater/docs/homestead-docker-build.md new file mode 100755 index 0000000..d3ceb5d --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docs/homestead-docker-build.md @@ -0,0 +1,1062 @@ + +Content + + [tangfx@localhost homer-docker]$ mkdir -p ../homestead-docker && cd ../homestead-docker + + [tangfx@localhost homestead-docker]$ vi Dockerfile + + [tangfx@localhost homestead-docker]$ vi Makefile + + [tangfx@localhost homestead-docker]$ make IMG_TAG=1611180614.gitrev-c2da9c9 + /usr/bin/docker build --tag=docker.io/tangfeixiong/clearwater-homestead:1611180614.gitrev-c2da9c9 --file=../../../../homestead/Dockerfile.tangfx ../../../../homestead + Sending build context to Docker daemon 8.192 kB + Step 1 : FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 + # Executing 2 build triggers... + Step 1 : EXPOSE 22 + ---> Using cache + Step 1 : CMD /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + ---> Using cache + ---> e16796a8d558 + Step 2 : MAINTAINER tangfeixiong + ---> Using cache + ---> e8f24f33160d + Step 3 : USER root + ---> Using cache + ---> dbc83c55f6a4 + Step 4 : ADD *supervisord.conf /etc/supervisor/conf.d/ + ---> f4beb208d7fe + Removing intermediate container 7737f2f027a7 + Step 5 : RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes homestead homestead-prov clearwater-prov-tools && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \1/g' -i /etc/init.d/homestead && mv /etc/supervisor/conf.d/homestead.supervisord.conf /etc/supervisor/conf.d/homestead.conf && mv /etc/supervisor/conf.d/homestead-prov.supervisord.conf /etc/supervisor/conf.d/homestead-prov.conf && mv /etc/supervisor/conf.d/nginx.supervisord.conf /etc/supervisor/conf.d/nginx.conf && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf + ---> Running in 04d75fd05f12 + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Ign http://mirrors.aliyun.com trusty InRelease + Get:1 http://mirrors.aliyun.com trusty-updates InRelease [65.9 kB] + Get:2 http://mirrors.aliyun.com trusty-security InRelease [65.9 kB] + Hit http://10.64.33.1:48080 binary/ Packages + Hit http://mirrors.aliyun.com trusty Release.gpg + Get:3 http://mirrors.aliyun.com trusty-updates/main Sources [476 kB] + Get:4 http://mirrors.aliyun.com trusty-updates/restricted Sources [5,921 B] + Get:5 http://mirrors.aliyun.com trusty-updates/universe Sources [214 kB] + Get:6 http://mirrors.aliyun.com trusty-updates/main amd64 Packages [1,145 kB] + Get:7 http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages [20.4 kB] + Get:8 http://mirrors.aliyun.com trusty-updates/universe amd64 Packages [502 kB] + Hit http://mirrors.aliyun.com trusty Release + Get:9 http://mirrors.aliyun.com trusty-security/main Sources [153 kB] + Get:10 http://mirrors.aliyun.com trusty-security/restricted Sources [4,621 B] + Get:11 http://mirrors.aliyun.com trusty-security/universe Sources [54.0 kB] + Get:12 http://mirrors.aliyun.com trusty-security/main amd64 Packages [681 kB] + Get:13 http://mirrors.aliyun.com trusty-security/restricted amd64 Packages [17.0 kB] + Get:14 http://mirrors.aliyun.com trusty-security/universe amd64 Packages [188 kB] + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Fetched 3,590 kB in 6s (575 kB/s) + Reading package lists... + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-updates_universe_binary-amd64_Packages.gz is not what the server reported 501609 501619 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_source_Sources.gz is not what the server reported 153056 153059 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_binary-amd64_Packages.gz is not what the server reported 680542 680554 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_universe_binary-amd64_Packages.gz is not what the server reported 187712 187715 + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + clearwater-log-cleanup clearwater-nginx clearwater-socket-factory + clearwater-tcp-scalability crest fontconfig-config fonts-dejavu-core + geoip-database homestead-libs libboost-filesystem1.54.0 libboost-regex1.54.0 + libboost-system1.54.0 libcurl3-gnutls libevent-2.0-5 libevent-core-2.0-5 + libevent-pthreads-2.0-5 libfontconfig1 libfreetype6 libgd3 libgeoip1 + libicu52 libjbig0 libjpeg-turbo8 libjpeg8 libpci3 libperl5.18 libsctp1 + libsnmp-base libsnmp30 libtiff5 libvpx1 libxml2 libxml2-dev libxpm4 + libxslt1-dev libxslt1.1 lksctp-tools nginx nginx-common nginx-core + python-zmq sgml-base xml-core + Suggested packages: + clearwater-snmp-handler-alarm clearwater-logging clearwater-snmpd + clearwater-secure-connections homestead-dbg clearwater-snmp-alarm-agent + libgd-tools geoip-bin snmp-mibs-downloader pkg-config fcgiwrap nginx-doc + sgml-base-doc debhelper + The following NEW packages will be installed: + clearwater-log-cleanup clearwater-nginx clearwater-prov-tools + clearwater-socket-factory clearwater-tcp-scalability crest fontconfig-config + fonts-dejavu-core geoip-database homestead homestead-libs homestead-prov + libboost-filesystem1.54.0 libboost-regex1.54.0 libboost-system1.54.0 + libcurl3-gnutls libevent-2.0-5 libevent-core-2.0-5 libevent-pthreads-2.0-5 + libfontconfig1 libfreetype6 libgd3 libgeoip1 libicu52 libjbig0 + libjpeg-turbo8 libjpeg8 libpci3 libperl5.18 libsctp1 libsnmp-base libsnmp30 + libtiff5 libvpx1 libxml2 libxml2-dev libxpm4 libxslt1-dev libxslt1.1 + lksctp-tools nginx nginx-common nginx-core python-zmq sgml-base xml-core + 0 upgraded, 46 newly installed, 0 to remove and 9 not upgraded. + Need to get 29.2 MB of archives. + After this operation, 111 MB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp-base 5.7.2~dfsg-clearwater4 [224 kB] + Get:2 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp30 5.7.2~dfsg-clearwater4 [811 kB] + Get:3 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-log-cleanup 1.0-161102.114310 [13.3 kB] + Get:4 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-nginx 1.0-161028.000011 [19.7 kB] + Get:5 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-prov-tools 1.0-161104.121210 [3,779 kB] + Get:6 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-socket-factory 1.0-161102.114310 [32.0 kB] + Get:7 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-tcp-scalability 1.0-161102.114310 [14.6 kB] + Get:8 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ crest 1.0-161104.120822 [9,294 kB] + Get:9 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libcurl3-gnutls amd64 7.35.0-1ubuntu2.10 [166 kB] + Get:10 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ homestead-libs 1.0-161104.122821 [1,006 kB] + Get:11 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ homestead 1.0-161104.122821 [380 kB] + Get:12 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ homestead-prov 1.0-161104.120822 [74.5 kB] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty/main libgeoip1 amd64 1.6.0-1 [71.0 kB] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpci3 amd64 1:3.2.1-1ubuntu5.1 [26.3 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libxml2 amd64 2.9.1+dfsg1-3ubuntu4.8 [573 kB] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty/main sgml-base all 1.26+nmu4ubuntu1 [12.5 kB] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-system1.54.0 amd64 1.54.0-4ubuntu3.1 [10.1 kB] + Get:18 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-filesystem1.54.0 amd64 1.54.0-4ubuntu3.1 [34.2 kB] + Get:19 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libicu52 amd64 52.1-3ubuntu0.4 [6,752 kB] + Get:20 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-regex1.54.0 amd64 1.54.0-4ubuntu3.1 [261 kB] + Get:21 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [126 kB] + Get:22 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-core-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [78.3 kB] + Get:23 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-pthreads-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [6,126 B] + Get:24 http://mirrors.aliyun.com/ubuntu/ trusty/main fonts-dejavu-core all 2.34-1ubuntu1 [1,024 kB] + Get:25 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main fontconfig-config all 2.11.0-0ubuntu4.2 [47.4 kB] + Get:26 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfreetype6 amd64 2.5.2-1ubuntu2.5 [304 kB] + Get:27 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfontconfig1 amd64 2.11.0-0ubuntu4.2 [123 kB] + Get:28 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg-turbo8 amd64 1.3.0-0ubuntu2 [104 kB] + Get:29 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg8 amd64 8c-2ubuntu8 [2,194 B] + Get:30 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libjbig0 amd64 2.0-2ubuntu4.1 [26.1 kB] + Get:31 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libtiff5 amd64 4.0.3-7ubuntu0.4 [143 kB] + Get:32 http://mirrors.aliyun.com/ubuntu/ trusty/main libvpx1 amd64 1.3.0-2 [556 kB] + Get:33 http://mirrors.aliyun.com/ubuntu/ trusty/main libxpm4 amd64 1:3.5.10-1 [38.3 kB] + Get:34 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgd3 amd64 2.1.0-3ubuntu0.5 [122 kB] + Get:35 http://mirrors.aliyun.com/ubuntu/ trusty/main libsctp1 amd64 1.0.15+dfsg-1 [9,226 B] + Get:36 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libperl5.18 amd64 5.18.2-2ubuntu1.1 [1,332 B] + Get:37 http://mirrors.aliyun.com/ubuntu/ trusty/main libxslt1.1 amd64 1.1.28-2build1 [145 kB] + Get:38 http://mirrors.aliyun.com/ubuntu/ trusty/main geoip-database all 20140313-1 [1,196 kB] + Get:39 http://mirrors.aliyun.com/ubuntu/ trusty/main xml-core all 0.13+nmu2 [23.3 kB] + Get:40 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-common all 1.4.6-1ubuntu3.7 [19.0 kB] + Get:41 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-core amd64 1.4.6-1ubuntu3.7 [325 kB] + Get:42 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx all 1.4.6-1ubuntu3.7 [5,352 B] + Get:43 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libxml2-dev amd64 2.9.1+dfsg1-3ubuntu4.8 [631 kB] + Get:44 http://mirrors.aliyun.com/ubuntu/ trusty/main libxslt1-dev amd64 1.1.28-2build1 [407 kB] + Get:45 http://mirrors.aliyun.com/ubuntu/ trusty/universe python-zmq amd64 14.0.1-1build2 [175 kB] + Get:46 http://mirrors.aliyun.com/ubuntu/ trusty/main lksctp-tools amd64 1.0.15+dfsg-1 [51.3 kB] + Extracting templates from packages: 100% + Preconfiguring packages ... + Fetched 29.2 MB in 13s (2,220 kB/s) + Selecting previously unselected package libcurl3-gnutls:amd64. + (Reading database ... 24489 files and directories currently installed.) + Preparing to unpack .../libcurl3-gnutls_7.35.0-1ubuntu2.10_amd64.deb ... + Unpacking libcurl3-gnutls:amd64 (7.35.0-1ubuntu2.10) ... + Selecting previously unselected package libgeoip1:amd64. + Preparing to unpack .../libgeoip1_1.6.0-1_amd64.deb ... + Unpacking libgeoip1:amd64 (1.6.0-1) ... + Selecting previously unselected package libpci3:amd64. + Preparing to unpack .../libpci3_1%3a3.2.1-1ubuntu5.1_amd64.deb ... + Unpacking libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Selecting previously unselected package libxml2:amd64. + Preparing to unpack .../libxml2_2.9.1+dfsg1-3ubuntu4.8_amd64.deb ... + Unpacking libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Selecting previously unselected package sgml-base. + Preparing to unpack .../sgml-base_1.26+nmu4ubuntu1_all.deb ... + Unpacking sgml-base (1.26+nmu4ubuntu1) ... + Selecting previously unselected package libboost-system1.54.0:amd64. + Preparing to unpack .../libboost-system1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libboost-filesystem1.54.0:amd64. + Preparing to unpack .../libboost-filesystem1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libicu52:amd64. + Preparing to unpack .../libicu52_52.1-3ubuntu0.4_amd64.deb ... + Unpacking libicu52:amd64 (52.1-3ubuntu0.4) ... + Selecting previously unselected package libboost-regex1.54.0:amd64. + Preparing to unpack .../libboost-regex1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libevent-2.0-5:amd64. + Preparing to unpack .../libevent-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-core-2.0-5:amd64. + Preparing to unpack .../libevent-core-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-pthreads-2.0-5:amd64. + Preparing to unpack .../libevent-pthreads-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package fonts-dejavu-core. + Preparing to unpack .../fonts-dejavu-core_2.34-1ubuntu1_all.deb ... + Unpacking fonts-dejavu-core (2.34-1ubuntu1) ... + Selecting previously unselected package fontconfig-config. + Preparing to unpack .../fontconfig-config_2.11.0-0ubuntu4.2_all.deb ... + Unpacking fontconfig-config (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libfreetype6:amd64. + Preparing to unpack .../libfreetype6_2.5.2-1ubuntu2.5_amd64.deb ... + Unpacking libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Selecting previously unselected package libfontconfig1:amd64. + Preparing to unpack .../libfontconfig1_2.11.0-0ubuntu4.2_amd64.deb ... + Unpacking libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libjpeg-turbo8:amd64. + Preparing to unpack .../libjpeg-turbo8_1.3.0-0ubuntu2_amd64.deb ... + Unpacking libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Selecting previously unselected package libjpeg8:amd64. + Preparing to unpack .../libjpeg8_8c-2ubuntu8_amd64.deb ... + Unpacking libjpeg8:amd64 (8c-2ubuntu8) ... + Selecting previously unselected package libjbig0:amd64. + Preparing to unpack .../libjbig0_2.0-2ubuntu4.1_amd64.deb ... + Unpacking libjbig0:amd64 (2.0-2ubuntu4.1) ... + Selecting previously unselected package libtiff5:amd64. + Preparing to unpack .../libtiff5_4.0.3-7ubuntu0.4_amd64.deb ... + Unpacking libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Selecting previously unselected package libvpx1:amd64. + Preparing to unpack .../libvpx1_1.3.0-2_amd64.deb ... + Unpacking libvpx1:amd64 (1.3.0-2) ... + Selecting previously unselected package libxpm4:amd64. + Preparing to unpack .../libxpm4_1%3a3.5.10-1_amd64.deb ... + Unpacking libxpm4:amd64 (1:3.5.10-1) ... + Selecting previously unselected package libgd3:amd64. + Preparing to unpack .../libgd3_2.1.0-3ubuntu0.5_amd64.deb ... + Unpacking libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Selecting previously unselected package libsctp1:amd64. + Preparing to unpack .../libsctp1_1.0.15+dfsg-1_amd64.deb ... + Unpacking libsctp1:amd64 (1.0.15+dfsg-1) ... + Selecting previously unselected package libperl5.18. + Preparing to unpack .../libperl5.18_5.18.2-2ubuntu1.1_amd64.deb ... + Unpacking libperl5.18 (5.18.2-2ubuntu1.1) ... + Selecting previously unselected package libsnmp-base. + Preparing to unpack .../libsnmp-base_5.7.2~dfsg-clearwater4_all.deb ... + Unpacking libsnmp-base (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package libsnmp30:amd64. + Preparing to unpack .../libsnmp30_5.7.2~dfsg-clearwater4_amd64.deb ... + Unpacking libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package libxslt1.1:amd64. + Preparing to unpack .../libxslt1.1_1.1.28-2build1_amd64.deb ... + Unpacking libxslt1.1:amd64 (1.1.28-2build1) ... + Selecting previously unselected package geoip-database. + Preparing to unpack .../geoip-database_20140313-1_all.deb ... + Unpacking geoip-database (20140313-1) ... + Selecting previously unselected package xml-core. + Preparing to unpack .../xml-core_0.13+nmu2_all.deb ... + Unpacking xml-core (0.13+nmu2) ... + Selecting previously unselected package clearwater-log-cleanup. + Preparing to unpack .../clearwater-log-cleanup_1.0-161102.114310_all.deb ... + Unpacking clearwater-log-cleanup (1.0-161102.114310) ... + Selecting previously unselected package nginx-common. + Preparing to unpack .../nginx-common_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx-common (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx-core. + Preparing to unpack .../nginx-core_1.4.6-1ubuntu3.7_amd64.deb ... + Unpacking nginx-core (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx. + Preparing to unpack .../nginx_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package clearwater-nginx. + Preparing to unpack .../clearwater-nginx_1.0-161028.000011_all.deb ... + Unpacking clearwater-nginx (1.0-161028.000011) ... + Selecting previously unselected package clearwater-prov-tools. + Preparing to unpack .../clearwater-prov-tools_1.0-161104.121210_amd64.deb ... + Unpacking clearwater-prov-tools (1.0-161104.121210) ... + Selecting previously unselected package clearwater-socket-factory. + Preparing to unpack .../clearwater-socket-factory_1.0-161102.114310_amd64.deb ... + Unpacking clearwater-socket-factory (1.0-161102.114310) ... + Selecting previously unselected package clearwater-tcp-scalability. + Preparing to unpack .../clearwater-tcp-scalability_1.0-161102.114310_all.deb ... + Unpacking clearwater-tcp-scalability (1.0-161102.114310) ... + Selecting previously unselected package libxml2-dev:amd64. + Preparing to unpack .../libxml2-dev_2.9.1+dfsg1-3ubuntu4.8_amd64.deb ... + Unpacking libxml2-dev:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Selecting previously unselected package libxslt1-dev:amd64. + Preparing to unpack .../libxslt1-dev_1.1.28-2build1_amd64.deb ... + Unpacking libxslt1-dev:amd64 (1.1.28-2build1) ... + Selecting previously unselected package python-zmq. + Preparing to unpack .../python-zmq_14.0.1-1build2_amd64.deb ... + Unpacking python-zmq (14.0.1-1build2) ... + Selecting previously unselected package crest. + Preparing to unpack .../crest_1.0-161104.120822_amd64.deb ... + Unpacking crest (1.0-161104.120822) ... + Selecting previously unselected package homestead-libs. + Preparing to unpack .../homestead-libs_1.0-161104.122821_amd64.deb ... + Unpacking homestead-libs (1.0-161104.122821) ... + Selecting previously unselected package homestead. + Preparing to unpack .../homestead_1.0-161104.122821_amd64.deb ... + Unpacking homestead (1.0-161104.122821) ... + Selecting previously unselected package homestead-prov. + Preparing to unpack .../homestead-prov_1.0-161104.120822_amd64.deb ... + Unpacking homestead-prov (1.0-161104.120822) ... + Selecting previously unselected package lksctp-tools. + Preparing to unpack .../lksctp-tools_1.0.15+dfsg-1_amd64.deb ... + Unpacking lksctp-tools (1.0.15+dfsg-1) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libcurl3-gnutls:amd64 (7.35.0-1ubuntu2.10) ... + Setting up libgeoip1:amd64 (1.6.0-1) ... + Setting up libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Setting up libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Setting up sgml-base (1.26+nmu4ubuntu1) ... + Setting up libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libicu52:amd64 (52.1-3ubuntu0.4) ... + Setting up libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up fonts-dejavu-core (2.34-1ubuntu1) ... + Setting up fontconfig-config (2.11.0-0ubuntu4.2) ... + Setting up libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Setting up libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Setting up libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Setting up libjpeg8:amd64 (8c-2ubuntu8) ... + Setting up libjbig0:amd64 (2.0-2ubuntu4.1) ... + Setting up libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Setting up libvpx1:amd64 (1.3.0-2) ... + Setting up libxpm4:amd64 (1:3.5.10-1) ... + Setting up libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Setting up libsctp1:amd64 (1.0.15+dfsg-1) ... + Setting up libperl5.18 (5.18.2-2ubuntu1.1) ... + Setting up libsnmp-base (5.7.2~dfsg-clearwater4) ... + Setting up libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Setting up libxslt1.1:amd64 (1.1.28-2build1) ... + Setting up geoip-database (20140313-1) ... + Setting up xml-core (0.13+nmu2) ... + Setting up clearwater-log-cleanup (1.0-161102.114310) ... + Setting up nginx-common (1.4.6-1ubuntu3.7) ... + Setting up clearwater-prov-tools (1.0-161104.121210) ... + Already using interpreter /usr/bin/python + New python executable in /usr/share/clearwater/clearwater-prov-tools/env/bin/python + Installing setuptools, pip...done. + Processing backports.ssl_match_hostname-3.5.0.1-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/backports.ssl_match_hostname-3.5.0.1-py2.7.egg + Extracting backports.ssl_match_hostname-3.5.0.1-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding backports.ssl-match-hostname 3.5.0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/backports.ssl_match_hostname-3.5.0.1-py2.7.egg + Processing dependencies for backports.ssl-match-hostname==3.5.0.1 + Finished processing dependencies for backports.ssl-match-hostname==3.5.0.1 + Processing backports_abc-0.4-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/backports_abc-0.4-py2.7.egg + Extracting backports_abc-0.4-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding backports-abc 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/backports_abc-0.4-py2.7.egg + Processing dependencies for backports-abc==0.4 + Finished processing dependencies for backports-abc==0.4 + Processing certifi-2016.9.26-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/certifi-2016.9.26-py2.7.egg + Extracting certifi-2016.9.26-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding certifi 2016.9.26 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/certifi-2016.9.26-py2.7.egg + Processing dependencies for certifi==2016.9.26 + Finished processing dependencies for certifi==2016.9.26 + Processing cffi-1.5.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Extracting cffi-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding cffi 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Processing dependencies for cffi==1.5.2 + Searching for pycparser + Best match: pycparser 2.17 + Processing pycparser-2.17-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Extracting pycparser-2.17-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding pycparser 2.17 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Finished processing dependencies for cffi==1.5.2 + Processing clearwater_prov_tools-0.1-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/clearwater_prov_tools-0.1-py2.7.egg + Extracting clearwater_prov_tools-0.1-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding clearwater-prov-tools 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/clearwater_prov_tools-0.1-py2.7.egg + Processing dependencies for clearwater-prov-tools==0.1 + Searching for tornado + Best match: tornado 4.4.2 + Processing tornado-4.4.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/tornado-4.4.2-py2.7-linux-x86_64.egg + Extracting tornado-4.4.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding tornado 4.4.2 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/tornado-4.4.2-py2.7-linux-x86_64.egg + Searching for singledispatch + Best match: singledispatch 3.4.0.3 + Processing singledispatch-3.4.0.3-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/singledispatch-3.4.0.3-py2.7.egg + Extracting singledispatch-3.4.0.3-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding singledispatch 3.4.0.3 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/singledispatch-3.4.0.3-py2.7.egg + Searching for six + Best match: six 1.10.0 + Processing six-1.10.0-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Extracting six-1.10.0-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding six 1.10.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Finished processing dependencies for clearwater-prov-tools==0.1 + Processing metaswitchcommon-0.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Extracting metaswitchcommon-0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding metaswitchcommon 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Processing dependencies for metaswitchcommon==0.1 + Searching for monotonic==0.6 + Best match: monotonic 0.6 + Processing monotonic-0.6-py2.7.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Extracting monotonic-0.6-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding monotonic 0.6 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Searching for pyzmq==15.2 + Best match: pyzmq 15.2.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Extracting pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + File "/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/asyncio.py", line 79 + "{!r}".format(fileobj)) from None + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/tests/_test_asyncio.py", line 43 + yield from a.send(b'hi') + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/auth/asyncio.py", line 28 + events = yield from self.__poller.poll() + ^ + SyntaxError: invalid syntax + + Adding pyzmq 15.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Searching for pycrypto==2.6.1 + Best match: pycrypto 2.6.1 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Extracting pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding pycrypto 2.6.1 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Searching for py-bcrypt + Best match: py-bcrypt 0.4 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Extracting py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + Adding py-bcrypt 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Finished processing dependencies for metaswitchcommon==0.1 + Processing monotonic-0.6-py2.7.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Extracting monotonic-0.6-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + monotonic 0.6 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Processing dependencies for monotonic==0.6 + Finished processing dependencies for monotonic==0.6 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Extracting py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + py-bcrypt 0.4 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Processing dependencies for py-bcrypt==0.4 + Finished processing dependencies for py-bcrypt==0.4 + Processing pycparser-2.17-py2.7.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Extracting pycparser-2.17-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + pycparser 2.17 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Processing dependencies for pycparser==2.17 + Finished processing dependencies for pycparser==2.17 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Extracting pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + pycrypto 2.6.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Processing dependencies for pycrypto==2.6.1 + Finished processing dependencies for pycrypto==2.6.1 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Extracting pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + File "/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/asyncio.py", line 79 + "{!r}".format(fileobj)) from None + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/tests/_test_asyncio.py", line 43 + yield from a.send(b'hi') + ^ + SyntaxError: invalid syntax + + File "/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg/zmq/auth/asyncio.py", line 28 + events = yield from self.__poller.poll() + ^ + SyntaxError: invalid syntax + + pyzmq 15.2.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Processing dependencies for pyzmq==15.2.0 + Finished processing dependencies for pyzmq==15.2.0 + Processing singledispatch-3.4.0.3-py2.7.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/singledispatch-3.4.0.3-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/singledispatch-3.4.0.3-py2.7.egg + Extracting singledispatch-3.4.0.3-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + singledispatch 3.4.0.3 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/singledispatch-3.4.0.3-py2.7.egg + Processing dependencies for singledispatch==3.4.0.3 + Finished processing dependencies for singledispatch==3.4.0.3 + Processing six-1.10.0-py2.7.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Extracting six-1.10.0-py2.7.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + six 1.10.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Processing dependencies for six==1.10.0 + Finished processing dependencies for six==1.10.0 + Processing tornado-4.4.2-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/tornado-4.4.2-py2.7-linux-x86_64.egg' (and everything under it) + creating /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/tornado-4.4.2-py2.7-linux-x86_64.egg + Extracting tornado-4.4.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages + tornado 4.4.2 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/clearwater-prov-tools/env/lib/python2.7/site-packages/tornado-4.4.2-py2.7-linux-x86_64.egg + Processing dependencies for tornado==4.4.2 + Finished processing dependencies for tornado==4.4.2 + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move 鈥▒/tmp/hosts.726鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + ** Note: Please use the --sec-param instead of --bits + Generating a 1024 bit RSA private key... + Generating a self signed certificate... + X.509 Certificate Information: + Version: 3 + Serial Number (hex): 582e4a7e + Validity: + Not Before: Fri Nov 18 00:25:34 UTC 2016 + Not After: Mon Nov 16 00:25:34 UTC 2026 + Subject: CN=10.120.160.2 + Subject Public Key Algorithm: RSA + Certificate Security Level: Weak + Modulus (bits 1024): + 00:d3:7f:03:f7:9c:be:88:e2:fe:c1:08:d4:96:50:fc + c9:16:18:1a:ec:52:18:cf:4a:de:a8:9b:34:d6:4b:0e + 12:33:03:9b:0b:d6:55:91:4b:24:43:a7:66:d9:17:d6 + 25:c5:b8:d7:e7:d9:67:06:4a:85:3e:c8:5d:22:b4:35 + f5:33:d6:2e:f5:a3:a9:58:fd:30:55:76:20:b4:d8:1e + 6a:f1:c3:c8:bf:18:d5:7d:19:74:cc:68:2e:85:fc:bf + f4:a3:6c:0b:35:d9:3f:61:96:d0:42:36:85:26:76:bf + bc:70:fa:f5:02:32:e2:27:3f:ef:cf:8c:da:05:04:06 + 43 + Exponent (bits 24): + 01:00:01 + Extensions: + Basic Constraints (critical): + Certificate Authority (CA): FALSE + Subject Key Identifier (not critical): + 953c7bb7ad808f82213400ea230f423bcaa82709 + Other Information: + Public Key Id: + 953c7bb7ad808f82213400ea230f423bcaa82709 + + + + Signing certificate... + hostname: you must be root to change the host name + mv: cannot move 鈥▒/tmp/hosts.854鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + Configuring monit for only localhost access + ...done. + Setting up clearwater-socket-factory (1.0-161102.114310) ... + Setting up clearwater-tcp-scalability (1.0-161102.114310) ... + Setting up libxml2-dev:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Setting up libxslt1-dev:amd64 (1.1.28-2build1) ... + Setting up python-zmq (14.0.1-1build2) ... + Setting up homestead-libs (1.0-161104.122821) ... + Setting up lksctp-tools (1.0.15+dfsg-1) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up nginx-core (1.4.6-1ubuntu3.7) ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up nginx (1.4.6-1ubuntu3.7) ... + Setting up clearwater-nginx (1.0-161028.000011) ... + Testing nginx configuration... + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + Site ping has been enabled. Run /etc/init.d/nginx reload to apply the changes. + Generating a 2048 bit RSA private key + .............................+++ + .................................+++ + writing new private key to 'nginx.key' + ----- + Signature ok + subject=/C=UK/ST=Unknown state/L=Unknown locality/O=Unknown organization/OU=Unknown organizational unit/CN=unknown.example/emailAddress=Unknown + Getting Private key + Adding 'diversion of /etc/nginx/nginx.conf to /etc/nginx/nginx.conf.clearwater-orig by clearwater-nginx' + Setting up crest (1.0-161104.120822) ... + Already using interpreter /usr/bin/python + New python executable in /usr/share/clearwater/crest/env/bin/python + Installing setuptools, pip...done. + Processing Twisted-16.5.0-py2.7-linux-x86_64.egg + Copying Twisted-16.5.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding Twisted 16.5.0 to easy-install.pth file + Installing trial script to /usr/share/clearwater/crest/env/bin + Installing conch script to /usr/share/clearwater/crest/env/bin + Installing ckeygen script to /usr/share/clearwater/crest/env/bin + Installing twist script to /usr/share/clearwater/crest/env/bin + Installing pyhtmlizer script to /usr/share/clearwater/crest/env/bin + Installing mailmail script to /usr/share/clearwater/crest/env/bin + Installing tkconch script to /usr/share/clearwater/crest/env/bin + Installing twistd script to /usr/share/clearwater/crest/env/bin + Installing cftp script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/Twisted-16.5.0-py2.7-linux-x86_64.egg + Processing dependencies for Twisted==16.5.0 + Searching for incremental>=16.10.1 + Best match: incremental 16.10.1 + Processing incremental-16.10.1-py2.7.egg + Copying incremental-16.10.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding incremental 16.10.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/incremental-16.10.1-py2.7.egg + Searching for constantly>=15.1 + Best match: constantly 15.1.0 + Processing constantly-15.1.0-py2.7.egg + Copying constantly-15.1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding constantly 15.1.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/constantly-15.1.0-py2.7.egg + Searching for zope.interface>=3.6.0 + Best match: zope.interface 4.3.2 + Processing zope.interface-4.3.2-py2.7-linux-x86_64.egg + Copying zope.interface-4.3.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding zope.interface 4.3.2 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/zope.interface-4.3.2-py2.7-linux-x86_64.egg + Finished processing dependencies for Twisted==16.5.0 + Processing cffi-1.5.2-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Extracting cffi-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cffi 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.5.2-py2.7-linux-x86_64.egg + Processing dependencies for cffi==1.5.2 + Searching for pycparser + Best match: pycparser 2.17 + Processing pycparser-2.17-py2.7.egg + Copying pycparser-2.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pycparser 2.17 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Finished processing dependencies for cffi==1.5.2 + Processing cffi-1.8.3-py2.7-linux-x86_64.egg + creating /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.8.3-py2.7-linux-x86_64.egg + Extracting cffi-1.8.3-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Removing cffi 1.5.2 from easy-install.pth file + Adding cffi 1.8.3 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cffi-1.8.3-py2.7-linux-x86_64.egg + Processing dependencies for cffi==1.8.3 + Finished processing dependencies for cffi==1.8.3 + Processing constantly-15.1.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/constantly-15.1.0-py2.7.egg + Copying constantly-15.1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + constantly 15.1.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/constantly-15.1.0-py2.7.egg + Processing dependencies for constantly==15.1.0 + Finished processing dependencies for constantly==15.1.0 + Processing cql-1.4.0-py2.7.egg + Copying cql-1.4.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cql 1.4.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cql-1.4.0-py2.7.egg + Processing dependencies for cql==1.4.0 + Searching for thrift + Best match: thrift 0.9.3 + Processing thrift-0.9.3-py2.7-linux-x86_64.egg + Copying thrift-0.9.3-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding thrift 0.9.3 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/thrift-0.9.3-py2.7-linux-x86_64.egg + Finished processing dependencies for cql==1.4.0 + Processing crest-0.1-py2.7.egg + creating /usr/share/clearwater/crest/env/lib/python2.7/site-packages/crest-0.1-py2.7.egg + Extracting crest-0.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding crest 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/crest-0.1-py2.7.egg + Processing dependencies for crest==0.1 + Searching for monotonic==0.6 + Best match: monotonic 0.6 + Processing monotonic-0.6-py2.7.egg + Copying monotonic-0.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding monotonic 0.6 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Searching for prctl + Best match: prctl 1.0.1 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + Copying prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding prctl 1.0.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Searching for pure-sasl + Best match: pure-sasl 0.3.0 + Processing pure_sasl-0.3.0-py2.7.egg + Copying pure_sasl-0.3.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pure-sasl 0.3.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pure_sasl-0.3.0-py2.7.egg + Searching for msgpack-python==0.4.7 + Best match: msgpack-python 0.4.7 + Processing msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Copying msgpack_python-0.4.7-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding msgpack-python 0.4.7 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Searching for lxml + Best match: lxml 3.6.4 + Processing lxml-3.6.4-py2.7-linux-x86_64.egg + Copying lxml-3.6.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding lxml 3.6.4 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/lxml-3.6.4-py2.7-linux-x86_64.egg + Searching for cyclone==1.0 + Best match: cyclone 1.0 + Processing cyclone-1.0-py2.7.egg + Copying cyclone-1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cyclone 1.0 to easy-install.pth file + Installing cyclone script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cyclone-1.0-py2.7.egg + Searching for py-bcrypt + Best match: py-bcrypt 0.4 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + Copying py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding py-bcrypt 0.4 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Searching for pyzmq==15.2 + Best match: pyzmq 15.2.0 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + Copying pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pyzmq 15.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Searching for pyopenssl + Best match: pyOpenSSL 16.2.0 + Processing pyOpenSSL-16.2.0-py2.7.egg + Copying pyOpenSSL-16.2.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pyOpenSSL 16.2.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyOpenSSL-16.2.0-py2.7.egg + Searching for six>=1.5.2 + Best match: six 1.10.0 + Processing six-1.10.0-py2.7.egg + Copying six-1.10.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding six 1.10.0 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Searching for cryptography>=1.3.4 + Best match: cryptography 1.5.2 + Processing cryptography-1.5.2-py2.7-linux-x86_64.egg + Copying cryptography-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding cryptography 1.5.2 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cryptography-1.5.2-py2.7-linux-x86_64.egg + Searching for ipaddress + Best match: ipaddress 1.0.17 + Processing ipaddress-1.0.17-py2.7.egg + Copying ipaddress-1.0.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding ipaddress 1.0.17 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/ipaddress-1.0.17-py2.7.egg + Searching for enum34 + Best match: enum34 1.1.6 + Processing enum34-1.1.6-py2.7.egg + Copying enum34-1.1.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding enum34 1.1.6 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/enum34-1.1.6-py2.7.egg + Searching for setuptools>=11.3 + Best match: setuptools 24.0.0 + Processing setuptools-24.0.0-py2.7.egg + Copying setuptools-24.0.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding setuptools 24.0.0 to easy-install.pth file + Installing easy_install script to /usr/share/clearwater/crest/env/bin + Installing easy_install-2.7 script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/setuptools-24.0.0-py2.7.egg + Searching for pyasn1>=0.1.8 + + Note: Bypassing https://pypi.python.org/packages/source/c/certifi/certifi-2016.2.28.tar.gz#md5=5d672aa766e1f773c75cfeccd02d3650 (disallowed host; see http://bit.ly/1dg9ijs for details). + + + Note: Bypassing https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2 (disallowed host; see http://bit.ly/1dg9ijs for details). + + Best match: pyasn1 0.1.9 + Processing pyasn1-0.1.9-py2.7.egg + Copying pyasn1-0.1.9-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pyasn1 0.1.9 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyasn1-0.1.9-py2.7.egg + Searching for idna>=2.0 + Best match: idna 2.1 + Processing idna-2.1-py2.7.egg + Copying idna-2.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding idna 2.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/idna-2.1-py2.7.egg + Finished processing dependencies for crest==0.1 + Processing cryptography-1.5.2-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/cryptography-1.5.2-py2.7-linux-x86_64.egg' (and everything under it) + Copying cryptography-1.5.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + cryptography 1.5.2 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cryptography-1.5.2-py2.7-linux-x86_64.egg + Processing dependencies for cryptography==1.5.2 + Finished processing dependencies for cryptography==1.5.2 + Processing cyclone-1.0-py2.7.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/cyclone-1.0-py2.7.egg' (and everything under it) + Copying cyclone-1.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + cyclone 1.0 is already the active version in easy-install.pth + Installing cyclone script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/cyclone-1.0-py2.7.egg + Processing dependencies for cyclone==1.0 + Finished processing dependencies for cyclone==1.0 + Processing enum34-1.1.6-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/enum34-1.1.6-py2.7.egg + Copying enum34-1.1.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + enum34 1.1.6 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/enum34-1.1.6-py2.7.egg + Processing dependencies for enum34==1.1.6 + Finished processing dependencies for enum34==1.1.6 + Processing idna-2.1-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/idna-2.1-py2.7.egg + Copying idna-2.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + idna 2.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/idna-2.1-py2.7.egg + Processing dependencies for idna==2.1 + Finished processing dependencies for idna==2.1 + Processing incremental-16.10.1-py2.7.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/incremental-16.10.1-py2.7.egg' (and everything under it) + Copying incremental-16.10.1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + incremental 16.10.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/incremental-16.10.1-py2.7.egg + Processing dependencies for incremental==16.10.1 + Finished processing dependencies for incremental==16.10.1 + Processing ipaddress-1.0.17-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/ipaddress-1.0.17-py2.7.egg + Copying ipaddress-1.0.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + ipaddress 1.0.17 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/ipaddress-1.0.17-py2.7.egg + Processing dependencies for ipaddress==1.0.17 + Finished processing dependencies for ipaddress==1.0.17 + Processing lxml-3.6.4-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/lxml-3.6.4-py2.7-linux-x86_64.egg' (and everything under it) + Copying lxml-3.6.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + lxml 3.6.4 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/lxml-3.6.4-py2.7-linux-x86_64.egg + Processing dependencies for lxml==3.6.4 + Finished processing dependencies for lxml==3.6.4 + Processing metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Copying metaswitchcommon-0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding metaswitchcommon 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/metaswitchcommon-0.1-py2.7-linux-x86_64.egg + Processing dependencies for metaswitchcommon==0.1 + Searching for pycrypto==2.6.1 + Best match: pycrypto 2.6.1 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + Copying pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding pycrypto 2.6.1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Finished processing dependencies for metaswitchcommon==0.1 + Processing monotonic-0.6-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Copying monotonic-0.6-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + monotonic 0.6 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/monotonic-0.6-py2.7.egg + Processing dependencies for monotonic==0.6 + Finished processing dependencies for monotonic==0.6 + Processing msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Copying msgpack_python-0.4.7-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + msgpack-python 0.4.7 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/msgpack_python-0.4.7-py2.7-linux-x86_64.egg + Processing dependencies for msgpack-python==0.4.7 + Finished processing dependencies for msgpack-python==0.4.7 + Processing prctl-1.0.1-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Copying prctl-1.0.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + prctl 1.0.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/prctl-1.0.1-py2.7-linux-x86_64.egg + Processing dependencies for prctl==1.0.1 + Finished processing dependencies for prctl==1.0.1 + Processing pure_sasl-0.3.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pure_sasl-0.3.0-py2.7.egg + Copying pure_sasl-0.3.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pure-sasl 0.3.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pure_sasl-0.3.0-py2.7.egg + Processing dependencies for pure-sasl==0.3.0 + Finished processing dependencies for pure-sasl==0.3.0 + Processing pyOpenSSL-16.2.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyOpenSSL-16.2.0-py2.7.egg + Copying pyOpenSSL-16.2.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pyOpenSSL 16.2.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyOpenSSL-16.2.0-py2.7.egg + Processing dependencies for pyOpenSSL==16.2.0 + Finished processing dependencies for pyOpenSSL==16.2.0 + Processing py_bcrypt-0.4-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Copying py_bcrypt-0.4-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + py-bcrypt 0.4 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/py_bcrypt-0.4-py2.7-linux-x86_64.egg + Processing dependencies for py-bcrypt==0.4 + Finished processing dependencies for py-bcrypt==0.4 + Processing pyasn1-0.1.9-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyasn1-0.1.9-py2.7.egg + Copying pyasn1-0.1.9-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pyasn1 0.1.9 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyasn1-0.1.9-py2.7.egg + Processing dependencies for pyasn1==0.1.9 + Finished processing dependencies for pyasn1==0.1.9 + Processing pycparser-2.17-py2.7.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg' (and everything under it) + Copying pycparser-2.17-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pycparser 2.17 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycparser-2.17-py2.7.egg + Processing dependencies for pycparser==2.17 + Finished processing dependencies for pycparser==2.17 + Processing pycrypto-2.6.1-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Copying pycrypto-2.6.1-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pycrypto 2.6.1 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pycrypto-2.6.1-py2.7-linux-x86_64.egg + Processing dependencies for pycrypto==2.6.1 + Finished processing dependencies for pycrypto==2.6.1 + Processing pyzmq-15.2.0-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg' (and everything under it) + Copying pyzmq-15.2.0-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + pyzmq 15.2.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/pyzmq-15.2.0-py2.7-linux-x86_64.egg + Processing dependencies for pyzmq==15.2.0 + Finished processing dependencies for pyzmq==15.2.0 + Processing setuptools-24.0.0-py2.7.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/setuptools-24.0.0-py2.7.egg + Copying setuptools-24.0.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + setuptools 24.0.0 is already the active version in easy-install.pth + Installing easy_install script to /usr/share/clearwater/crest/env/bin + Installing easy_install-2.7 script to /usr/share/clearwater/crest/env/bin + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/setuptools-24.0.0-py2.7.egg + Processing dependencies for setuptools==24.0.0 + Finished processing dependencies for setuptools==24.0.0 + Processing six-1.10.0-py2.7.egg + + Note: Bypassing https://pypi.python.org/packages/source/c/certifi/certifi-2016.2.28.tar.gz#md5=5d672aa766e1f773c75cfeccd02d3650 (disallowed host; see http://bit.ly/1dg9ijs for details). + + + Note: Bypassing https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2 (disallowed host; see http://bit.ly/1dg9ijs for details). + + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg' (and everything under it) + Copying six-1.10.0-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + six 1.10.0 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/six-1.10.0-py2.7.egg + Processing dependencies for six==1.10.0 + Finished processing dependencies for six==1.10.0 + Processing telephus-1.0.0b1-py2.7.egg + Copying telephus-1.0.0b1-py2.7.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + Adding telephus 1.0.0b1 to easy-install.pth file + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/telephus-1.0.0b1-py2.7.egg + Processing dependencies for telephus==1.0.0b1 + Finished processing dependencies for telephus==1.0.0b1 + Processing thrift-0.9.3-py2.7-linux-x86_64.egg + Removing /usr/share/clearwater/crest/env/lib/python2.7/site-packages/thrift-0.9.3-py2.7-linux-x86_64.egg + Copying thrift-0.9.3-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + thrift 0.9.3 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/thrift-0.9.3-py2.7-linux-x86_64.egg + Processing dependencies for thrift==0.9.3 + Finished processing dependencies for thrift==0.9.3 + Processing zope.interface-4.3.2-py2.7-linux-x86_64.egg + removing '/usr/share/clearwater/crest/env/lib/python2.7/site-packages/zope.interface-4.3.2-py2.7-linux-x86_64.egg' (and everything under it) + Copying zope.interface-4.3.2-py2.7-linux-x86_64.egg to /usr/share/clearwater/crest/env/lib/python2.7/site-packages + zope.interface 4.3.2 is already the active version in easy-install.pth + + Installed /usr/share/clearwater/crest/env/lib/python2.7/site-packages/zope.interface-4.3.2-py2.7-linux-x86_64.egg + Processing dependencies for zope.interface==4.3.2 + Finished processing dependencies for zope.interface==4.3.2 + Setting up homestead (1.0-161104.122821) ... + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move 鈥▒/tmp/hosts.991鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + hostname: you must be root to change the host name + mv: cannot move 鈥▒/tmp/hosts.1100鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + Configuring monit for only localhost access + ...done. + Setting up homestead-prov (1.0-161104.120822) ... + Creating /usr/share/clearwater/homestead/python/packages/site.py + Processing homestead_prov-0.1-py2.7.egg + creating /usr/share/clearwater/homestead/python/packages/homestead_prov-0.1-py2.7.egg + Extracting homestead_prov-0.1-py2.7.egg to /usr/share/clearwater/homestead/python/packages + Adding homestead-prov 0.1 to easy-install.pth file + + Installed /usr/share/clearwater/homestead/python/packages/homestead_prov-0.1-py2.7.egg + Processing dependencies for homestead-prov==0.1 + Finished processing dependencies for homestead-prov==0.1 + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move 鈥▒/tmp/hosts.1137鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + hostname: you must be root to change the host name + mv: cannot move 鈥▒/tmp/hosts.1246鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + Configuring monit for only localhost access + ...done. + /var/lib/dpkg/info/homestead-prov.postinst: 69: /var/lib/dpkg/info/homestead-prov.postinst: /usr/share/clearwater/cassandra-schemas/homestead_provisioning.sh: not found + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + Processing triggers for sgml-base (1.26+nmu4ubuntu1) ... + Processing triggers for ureadahead (0.100.0-16) ... + ---> e5854d16214e + Removing intermediate container 04d75fd05f12 + Step 6 : EXPOSE 8888 8889 + ---> Running in e1ffa240a3ab + ---> 63ace52bb817 + Removing intermediate container e1ffa240a3ab + Successfully built 63ace52bb817 diff --git a/hack/kubernetes-clearwater/clearwater/docs/ralf-docker-build.md b/hack/kubernetes-clearwater/clearwater/docs/ralf-docker-build.md new file mode 100755 index 0000000..8b48ecc --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docs/ralf-docker-build.md @@ -0,0 +1,236 @@ + +Content + + [tangfx@localhost homestead-docker]$ mkdir -p ../ralf-docker && cd ../ralf-docker + + [tangfx@localhost ralf-docker]$ vi Dockerfile + + [tangfx@localhost ralf-docker]$ vi Makefile + + [tangfx@localhost ralf]$ make IMG_TAG=1611180614.gitrev-c2da9c9 + /usr/bin/docker build --tag=docker.io/tangfeixiong/clearwater-ralf:1611180614.gitrev-c2da9c9 --file=../../../../ralf/Dockerfile.tangfx ../../../../ralf + Sending build context to Docker daemon 23.55 kB + Step 1 : FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 + # Executing 2 build triggers... + Step 1 : EXPOSE 22 + ---> Using cache + Step 1 : CMD /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + ---> Using cache + ---> e16796a8d558 + Step 2 : MAINTAINER tangfeixiong + ---> Using cache + ---> e8f24f33160d + Step 3 : USER root + ---> Using cache + ---> dbc83c55f6a4 + Step 4 : ADD *supervisord.conf /etc/supervisor/conf.d/ + ---> 24121aac3fa7 + Removing intermediate container 9011c403cce5 + Step 5 : RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes ralf && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/ralf && mv /etc/supervisor/conf.d/ralf.supervisord.conf /etc/supervisor/conf.d/ralf.conf && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf + ---> Running in 84eb2fb090e2 + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Ign http://mirrors.aliyun.com trusty InRelease + Get:1 http://mirrors.aliyun.com trusty-updates InRelease [65.9 kB] + Get:2 http://mirrors.aliyun.com trusty-security InRelease [65.9 kB] + Hit http://10.64.33.1:48080 binary/ Packages + Hit http://mirrors.aliyun.com trusty Release.gpg + Get:3 http://mirrors.aliyun.com trusty-updates/main Sources [476 kB] + Get:4 http://mirrors.aliyun.com trusty-updates/restricted Sources [5,921 B] + Get:5 http://mirrors.aliyun.com trusty-updates/universe Sources [214 kB] + Get:6 http://mirrors.aliyun.com trusty-updates/main amd64 Packages [1,145 kB] + Get:7 http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages [20.4 kB] + Get:8 http://mirrors.aliyun.com trusty-updates/universe amd64 Packages [502 kB] + Hit http://mirrors.aliyun.com trusty Release + Get:9 http://mirrors.aliyun.com trusty-security/main Sources [153 kB] + Get:10 http://mirrors.aliyun.com trusty-security/restricted Sources [4,621 B] + Get:11 http://mirrors.aliyun.com trusty-security/universe Sources [54.0 kB] + Get:12 http://mirrors.aliyun.com trusty-security/main amd64 Packages [681 kB] + Get:13 http://mirrors.aliyun.com trusty-security/restricted amd64 Packages [17.0 kB] + Get:14 http://mirrors.aliyun.com trusty-security/universe amd64 Packages [188 kB] + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Fetched 3,590 kB in 6s (558 kB/s) + Reading package lists... + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-updates_main_source_Sources.gz is not what the server reported 475624 475632 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_source_Sources.gz is not what the server reported 153056 153059 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_binary-amd64_Packages.gz is not what the server reported 680542 680554 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_universe_binary-amd64_Packages.gz is not what the server reported 187712 187715 + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + clearwater-log-cleanup clearwater-socket-factory clearwater-tcp-scalability + libboost-filesystem1.54.0 libboost-regex1.54.0 libboost-system1.54.0 + libevent-2.0-5 libevent-core-2.0-5 libevent-pthreads-2.0-5 libicu52 libpci3 + libperl5.18 libsctp1 libsnmp-base libsnmp30 lksctp-tools ralf-libs + Suggested packages: + snmp-mibs-downloader ralf-dbg clearwater-snmp-alarm-agent + The following NEW packages will be installed: + clearwater-log-cleanup clearwater-socket-factory clearwater-tcp-scalability + libboost-filesystem1.54.0 libboost-regex1.54.0 libboost-system1.54.0 + libevent-2.0-5 libevent-core-2.0-5 libevent-pthreads-2.0-5 libicu52 libpci3 + libperl5.18 libsctp1 libsnmp-base libsnmp30 lksctp-tools ralf ralf-libs + 0 upgraded, 18 newly installed, 0 to remove and 9 not upgraded. + Need to get 9,421 kB of archives. + After this operation, 38.5 MB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp-base 5.7.2~dfsg-clearwater4 [224 kB] + Get:2 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp30 5.7.2~dfsg-clearwater4 [811 kB] + Get:3 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-log-cleanup 1.0-161102.114310 [13.3 kB] + Get:4 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-socket-factory 1.0-161102.114310 [32.0 kB] + Get:5 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-tcp-scalability 1.0-161102.114310 [14.6 kB] + Get:6 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ ralf-libs 1.0-161104.122049 [656 kB] + Get:7 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ ralf 1.0-161104.122049 [314 kB] + Get:8 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpci3 amd64 1:3.2.1-1ubuntu5.1 [26.3 kB] + Get:9 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-system1.54.0 amd64 1.54.0-4ubuntu3.1 [10.1 kB] + Get:10 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-filesystem1.54.0 amd64 1.54.0-4ubuntu3.1 [34.2 kB] + Get:11 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libicu52 amd64 52.1-3ubuntu0.4 [6,752 kB] + Get:12 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-regex1.54.0 amd64 1.54.0-4ubuntu3.1 [261 kB] + Get:13 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [126 kB] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-core-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [78.3 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-pthreads-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [6,126 B] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty/main libsctp1 amd64 1.0.15+dfsg-1 [9,226 B] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libperl5.18 amd64 5.18.2-2ubuntu1.1 [1,332 B] + Get:18 http://mirrors.aliyun.com/ubuntu/ trusty/main lksctp-tools amd64 1.0.15+dfsg-1 [51.3 kB] + Fetched 9,421 kB in 12s (741 kB/s) + Selecting previously unselected package libpci3:amd64. + (Reading database ... 24489 files and directories currently installed.) + Preparing to unpack .../libpci3_1%3a3.2.1-1ubuntu5.1_amd64.deb ... + Unpacking libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Selecting previously unselected package libboost-system1.54.0:amd64. + Preparing to unpack .../libboost-system1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libboost-filesystem1.54.0:amd64. + Preparing to unpack .../libboost-filesystem1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libicu52:amd64. + Preparing to unpack .../libicu52_52.1-3ubuntu0.4_amd64.deb ... + Unpacking libicu52:amd64 (52.1-3ubuntu0.4) ... + Selecting previously unselected package libboost-regex1.54.0:amd64. + Preparing to unpack .../libboost-regex1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libevent-2.0-5:amd64. + Preparing to unpack .../libevent-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-core-2.0-5:amd64. + Preparing to unpack .../libevent-core-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-pthreads-2.0-5:amd64. + Preparing to unpack .../libevent-pthreads-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libsctp1:amd64. + Preparing to unpack .../libsctp1_1.0.15+dfsg-1_amd64.deb ... + Unpacking libsctp1:amd64 (1.0.15+dfsg-1) ... + Selecting previously unselected package libperl5.18. + Preparing to unpack .../libperl5.18_5.18.2-2ubuntu1.1_amd64.deb ... + Unpacking libperl5.18 (5.18.2-2ubuntu1.1) ... + Selecting previously unselected package libsnmp-base. + Preparing to unpack .../libsnmp-base_5.7.2~dfsg-clearwater4_all.deb ... + Unpacking libsnmp-base (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package libsnmp30:amd64. + Preparing to unpack .../libsnmp30_5.7.2~dfsg-clearwater4_amd64.deb ... + Unpacking libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package clearwater-log-cleanup. + Preparing to unpack .../clearwater-log-cleanup_1.0-161102.114310_all.deb ... + Unpacking clearwater-log-cleanup (1.0-161102.114310) ... + Selecting previously unselected package clearwater-socket-factory. + Preparing to unpack .../clearwater-socket-factory_1.0-161102.114310_amd64.deb ... + Unpacking clearwater-socket-factory (1.0-161102.114310) ... + Selecting previously unselected package clearwater-tcp-scalability. + Preparing to unpack .../clearwater-tcp-scalability_1.0-161102.114310_all.deb ... + Unpacking clearwater-tcp-scalability (1.0-161102.114310) ... + Selecting previously unselected package lksctp-tools. + Preparing to unpack .../lksctp-tools_1.0.15+dfsg-1_amd64.deb ... + Unpacking lksctp-tools (1.0.15+dfsg-1) ... + Selecting previously unselected package ralf-libs. + Preparing to unpack .../ralf-libs_1.0-161104.122049_amd64.deb ... + Unpacking ralf-libs (1.0-161104.122049) ... + Selecting previously unselected package ralf. + Preparing to unpack .../ralf_1.0-161104.122049_amd64.deb ... + Unpacking ralf (1.0-161104.122049) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Setting up libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libicu52:amd64 (52.1-3ubuntu0.4) ... + Setting up libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libsctp1:amd64 (1.0.15+dfsg-1) ... + Setting up libperl5.18 (5.18.2-2ubuntu1.1) ... + Setting up libsnmp-base (5.7.2~dfsg-clearwater4) ... + Setting up libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Setting up clearwater-log-cleanup (1.0-161102.114310) ... + Setting up clearwater-socket-factory (1.0-161102.114310) ... + Setting up clearwater-tcp-scalability (1.0-161102.114310) ... + Setting up lksctp-tools (1.0.15+dfsg-1) ... + Setting up ralf-libs (1.0-161104.122049) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up ralf (1.0-161104.122049) ... + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move 鈥▒/tmp/hosts.305鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + hostname: you must be root to change the host name + mv: cannot move 鈥▒/tmp/hosts.326鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + Configuring monit for only localhost access + ** Note: Please use the --sec-param instead of --bits + Generating a 1024 bit RSA private key... + Generating a self signed certificate... + X.509 Certificate Information: + Version: 3 + Serial Number (hex): 582e4e22 + Validity: + Not Before: Fri Nov 18 00:41:06 UTC 2016 + Not After: Mon Nov 16 00:41:06 UTC 2026 + Subject: CN=10.120.160.2 + Subject Public Key Algorithm: RSA + Certificate Security Level: Weak + Modulus (bits 1024): + 00:db:17:49:39:b6:71:54:72:21:8d:fd:1e:83:1b:70 + 49:85:9c:48:7d:b4:24:0a:09:58:2b:e4:b6:3f:9b:8a + db:b9:ea:3f:7c:77:d7:38:92:9d:46:31:7d:d7:91:c6 + 57:d0:05:e1:82:fe:df:f6:c6:24:12:ee:73:9b:4d:8e + 41:97:80:fd:34:20:b2:09:29:a8:31:61:b3:e2:76:a9 + d2:9e:27:18:c3:8c:4f:62:57:10:f8:60:48:27:f7:0e + 07:24:9e:37:1f:94:57:46:34:95:48:d7:6f:40:2e:eb + a4:3f:69:e7:bd:57:af:4a:3c:0b:97:86:3b:44:40:b5 + 0d + Exponent (bits 24): + 01:00:01 + Extensions: + Basic Constraints (critical): + Certificate Authority (CA): FALSE + Subject Key Identifier (not critical): + f4c5e10b23a65ece1c8e884e6ab31d982c601ee9 + Other Information: + Public Key Id: + f4c5e10b23a65ece1c8e884e6ab31d982c601ee9 + + + + Signing certificate... + ...done. + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + Processing triggers for ureadahead (0.100.0-16) ... + ---> a86617a0752b + Removing intermediate container 84eb2fb090e2 + Step 6 : ADD plugins/* /usr/share/clearwater/clearwater-cluster-manager/plugins/ + ---> e9d9376ca31e + Removing intermediate container 345caf7ca321 + Step 7 : ADD reload_memcached_users /usr/share/clearwater/bin/reload_memcached_users + ---> 01b340857f6d + Removing intermediate container a4cde614b5cd + Step 8 : EXPOSE 10888 + ---> Running in 530c6baf530c + ---> a7181ba36557 + Removing intermediate container 530c6baf530c + Successfully built a7181ba36557 diff --git a/hack/kubernetes-clearwater/clearwater/docs/sprout-docker-build.md b/hack/kubernetes-clearwater/clearwater/docs/sprout-docker-build.md new file mode 100755 index 0000000..d74ba0e --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/docs/sprout-docker-build.md @@ -0,0 +1,359 @@ + +Content + + [tangfx@localhost ralf]$ mkdir -p ../sprout-docker && cd ../sprout-docker + + [tangfx@localhost sprout-docker]$ vi Dockerfile + + [tangfx@localhost sprout-docker]$ vi Makefile + + [tangfx@localhost sprout-docker]$ make IMG_TAG=1611180614.gitrev-c2da9c9 + /usr/bin/docker build --tag=docker.io/tangfeixiong/clearwater-sprout:1611180614.gitrev-c2da9c9 --file=../../../../sprout/Dockerfile.tangfx ../../../../sprout + Sending build context to Docker daemon 24.06 kB + Step 1 : FROM docker.io/tangfeixiong/clearwater-base-onbuild:1611180614.gitrev-c2da9c9 + # Executing 2 build triggers... + Step 1 : EXPOSE 22 + ---> Using cache + Step 1 : CMD /usr/bin/supervisord -c /etc/supervisor/supervisord.conf + ---> Using cache + ---> e16796a8d558 + Step 2 : MAINTAINER tangfeixiong + ---> Using cache + ---> e8f24f33160d + Step 3 : USER root + ---> Using cache + ---> dbc83c55f6a4 + Step 4 : ADD *supervisord.conf /etc/supervisor/conf.d/ + ---> Using cache + ---> 1faea0b5bb7c + Step 5 : RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes sprout && sed -e 's/\(echo 0 > \/proc\/sys\/kernel\/yama\/ptrace_scope\)/# \0/g' -i /etc/init.d/sprout && mv /etc/supervisor/conf.d/sprout.supervisord.conf /etc/supervisor/conf.d/sprout.conf && mv /etc/supervisor/conf.d/clearwater-group.supervisord.conf /etc/supervisor/conf.d/clearwater-group.conf + ---> Running in b9ae61eb4e92 + Ign http://10.64.33.1:48080 binary/ InRelease + Hit http://10.64.33.1:48080 binary/ Release.gpg + Hit http://10.64.33.1:48080 binary/ Release + Ign http://mirrors.aliyun.com trusty InRelease + Get:1 http://mirrors.aliyun.com trusty-updates InRelease [65.9 kB] + Get:2 http://mirrors.aliyun.com trusty-security InRelease [65.9 kB] + Hit http://10.64.33.1:48080 binary/ Packages + Hit http://mirrors.aliyun.com trusty Release.gpg + Get:3 http://mirrors.aliyun.com trusty-updates/main Sources [476 kB] + Get:4 http://mirrors.aliyun.com trusty-updates/restricted Sources [5,921 B] + Get:5 http://mirrors.aliyun.com trusty-updates/universe Sources [214 kB] + Get:6 http://mirrors.aliyun.com trusty-updates/main amd64 Packages [1,145 kB] + Get:7 http://mirrors.aliyun.com trusty-updates/restricted amd64 Packages [20.4 kB] + Get:8 http://mirrors.aliyun.com trusty-updates/universe amd64 Packages [502 kB] + Hit http://mirrors.aliyun.com trusty Release + Get:9 http://mirrors.aliyun.com trusty-security/main Sources [153 kB] + Get:10 http://mirrors.aliyun.com trusty-security/restricted Sources [4,621 B] + Get:11 http://mirrors.aliyun.com trusty-security/universe Sources [54.0 kB] + Get:12 http://mirrors.aliyun.com trusty-security/main amd64 Packages [681 kB] + Get:13 http://mirrors.aliyun.com trusty-security/restricted amd64 Packages [17.0 kB] + Get:14 http://mirrors.aliyun.com trusty-security/universe amd64 Packages [188 kB] + Hit http://mirrors.aliyun.com trusty/main Sources + Hit http://mirrors.aliyun.com trusty/restricted Sources + Hit http://mirrors.aliyun.com trusty/universe Sources + Hit http://mirrors.aliyun.com trusty/main amd64 Packages + Hit http://mirrors.aliyun.com trusty/restricted amd64 Packages + Hit http://mirrors.aliyun.com trusty/universe amd64 Packages + Fetched 3,590 kB in 5s (630 kB/s) + Reading package lists... + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-updates_universe_binary-amd64_Packages.gz is not what the server reported 501609 501619 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_source_Sources.gz is not what the server reported 153056 153059 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_main_binary-amd64_Packages.gz is not what the server reported 680542 680554 + W: Size of file /var/lib/apt/lists/mirrors.aliyun.com_ubuntu_dists_trusty-security_universe_binary-amd64_Packages.gz is not what the server reported 187712 187715 + Reading package lists... + Building dependency tree... + Reading state information... + The following extra packages will be installed: + clearwater-log-cleanup clearwater-nginx clearwater-socket-factory + clearwater-tcp-scalability fontconfig-config fonts-dejavu-core + geoip-database libboost-filesystem1.54.0 libboost-regex1.54.0 + libboost-system1.54.0 libboost-thread1.54.0 libevent-2.0-5 + libevent-core-2.0-5 libevent-pthreads-2.0-5 libfontconfig1 libfreetype6 + libgd3 libgeoip1 libicu52 libjbig0 libjpeg-turbo8 libjpeg8 libpci3 + libperl5.18 libsnmp-base libsnmp30 libtiff5 libvpx1 libxml2 libxpm4 + libxslt1.1 nginx nginx-common nginx-core sgml-base sprout-base sprout-bgcf + sprout-icscf sprout-libs sprout-mmtel-as sprout-scscf xml-core + Suggested packages: + clearwater-snmp-handler-alarm libgd-tools geoip-bin snmp-mibs-downloader + fcgiwrap nginx-doc sgml-base-doc sprout-dbg clearwater-logging + clearwater-snmp-alarm-agent sprout-bgcf-dbg sprout-icscf-dbg + sprout-mmtel-as-dbg sprout-scscf-dbg debhelper + The following NEW packages will be installed: + clearwater-log-cleanup clearwater-nginx clearwater-socket-factory + clearwater-tcp-scalability fontconfig-config fonts-dejavu-core + geoip-database libboost-filesystem1.54.0 libboost-regex1.54.0 + libboost-system1.54.0 libboost-thread1.54.0 libevent-2.0-5 + libevent-core-2.0-5 libevent-pthreads-2.0-5 libfontconfig1 libfreetype6 + libgd3 libgeoip1 libicu52 libjbig0 libjpeg-turbo8 libjpeg8 libpci3 + libperl5.18 libsnmp-base libsnmp30 libtiff5 libvpx1 libxml2 libxpm4 + libxslt1.1 nginx nginx-common nginx-core sgml-base sprout sprout-base + sprout-bgcf sprout-icscf sprout-libs sprout-mmtel-as sprout-scscf xml-core + 0 upgraded, 43 newly installed, 0 to remove and 9 not upgraded. + Need to get 16.7 MB of archives. + After this operation, 63.8 MB of additional disk space will be used. + Get:1 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp-base 5.7.2~dfsg-clearwater4 [224 kB] + Get:2 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ libsnmp30 5.7.2~dfsg-clearwater4 [811 kB] + Get:3 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-log-cleanup 1.0-161102.114310 [13.3 kB] + Get:4 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-nginx 1.0-161028.000011 [19.7 kB] + Get:5 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-socket-factory 1.0-161102.114310 [32.0 kB] + Get:6 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ clearwater-tcp-scalability 1.0-161102.114310 [14.6 kB] + Get:7 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout-libs 1.0-161104.171821 [1,551 kB] + Get:8 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout-base 1.0-161104.171821 [1,571 kB] + Get:9 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout-scscf 1.0-161104.171821 [60.5 kB] + Get:10 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout-icscf 1.0-161104.171821 [73.8 kB] + Get:11 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout-bgcf 1.0-161104.171821 [62.2 kB] + Get:12 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout-mmtel-as 1.0-161104.171821 [46.2 kB] + Get:13 http://10.64.33.1:48080/mirror/repo.cw-ngv.com/stable/ binary/ sprout 1.0-161104.171821 [12.4 kB] + Get:14 http://mirrors.aliyun.com/ubuntu/ trusty/main libgeoip1 amd64 1.6.0-1 [71.0 kB] + Get:15 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libpci3 amd64 1:3.2.1-1ubuntu5.1 [26.3 kB] + Get:16 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libxml2 amd64 2.9.1+dfsg1-3ubuntu4.8 [573 kB] + Get:17 http://mirrors.aliyun.com/ubuntu/ trusty/main sgml-base all 1.26+nmu4ubuntu1 [12.5 kB] + Get:18 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-system1.54.0 amd64 1.54.0-4ubuntu3.1 [10.1 kB] + Get:19 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-filesystem1.54.0 amd64 1.54.0-4ubuntu3.1 [34.2 kB] + Get:20 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libicu52 amd64 52.1-3ubuntu0.4 [6,752 kB] + Get:21 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-regex1.54.0 amd64 1.54.0-4ubuntu3.1 [261 kB] + Get:22 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libboost-thread1.54.0 amd64 1.54.0-4ubuntu3.1 [26.5 kB] + Get:23 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [126 kB] + Get:24 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-core-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [78.3 kB] + Get:25 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libevent-pthreads-2.0-5 amd64 2.0.21-stable-1ubuntu1.14.04.1 [6,126 B] + Get:26 http://mirrors.aliyun.com/ubuntu/ trusty/main fonts-dejavu-core all 2.34-1ubuntu1 [1,024 kB] + Get:27 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main fontconfig-config all 2.11.0-0ubuntu4.2 [47.4 kB] + Get:28 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfreetype6 amd64 2.5.2-1ubuntu2.5 [304 kB] + Get:29 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libfontconfig1 amd64 2.11.0-0ubuntu4.2 [123 kB] + Get:30 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg-turbo8 amd64 1.3.0-0ubuntu2 [104 kB] + Get:31 http://mirrors.aliyun.com/ubuntu/ trusty/main libjpeg8 amd64 8c-2ubuntu8 [2,194 B] + Get:32 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libjbig0 amd64 2.0-2ubuntu4.1 [26.1 kB] + Get:33 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libtiff5 amd64 4.0.3-7ubuntu0.4 [143 kB] + Get:34 http://mirrors.aliyun.com/ubuntu/ trusty/main libvpx1 amd64 1.3.0-2 [556 kB] + Get:35 http://mirrors.aliyun.com/ubuntu/ trusty/main libxpm4 amd64 1:3.5.10-1 [38.3 kB] + Get:36 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libgd3 amd64 2.1.0-3ubuntu0.5 [122 kB] + Get:37 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main libperl5.18 amd64 5.18.2-2ubuntu1.1 [1,332 B] + Get:38 http://mirrors.aliyun.com/ubuntu/ trusty/main libxslt1.1 amd64 1.1.28-2build1 [145 kB] + Get:39 http://mirrors.aliyun.com/ubuntu/ trusty/main geoip-database all 20140313-1 [1,196 kB] + Get:40 http://mirrors.aliyun.com/ubuntu/ trusty/main xml-core all 0.13+nmu2 [23.3 kB] + Get:41 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-common all 1.4.6-1ubuntu3.7 [19.0 kB] + Get:42 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx-core amd64 1.4.6-1ubuntu3.7 [325 kB] + Get:43 http://mirrors.aliyun.com/ubuntu/ trusty-updates/main nginx all 1.4.6-1ubuntu3.7 [5,352 B] + Extracting templates from packages: 100% + Preconfiguring packages ... + Fetched 16.7 MB in 9s (1,670 kB/s) + Selecting previously unselected package libgeoip1:amd64. + (Reading database ... 24489 files and directories currently installed.) + Preparing to unpack .../libgeoip1_1.6.0-1_amd64.deb ... + Unpacking libgeoip1:amd64 (1.6.0-1) ... + Selecting previously unselected package libpci3:amd64. + Preparing to unpack .../libpci3_1%3a3.2.1-1ubuntu5.1_amd64.deb ... + Unpacking libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Selecting previously unselected package libxml2:amd64. + Preparing to unpack .../libxml2_2.9.1+dfsg1-3ubuntu4.8_amd64.deb ... + Unpacking libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Selecting previously unselected package sgml-base. + Preparing to unpack .../sgml-base_1.26+nmu4ubuntu1_all.deb ... + Unpacking sgml-base (1.26+nmu4ubuntu1) ... + Selecting previously unselected package libboost-system1.54.0:amd64. + Preparing to unpack .../libboost-system1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libboost-filesystem1.54.0:amd64. + Preparing to unpack .../libboost-filesystem1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libicu52:amd64. + Preparing to unpack .../libicu52_52.1-3ubuntu0.4_amd64.deb ... + Unpacking libicu52:amd64 (52.1-3ubuntu0.4) ... + Selecting previously unselected package libboost-regex1.54.0:amd64. + Preparing to unpack .../libboost-regex1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libboost-thread1.54.0:amd64. + Preparing to unpack .../libboost-thread1.54.0_1.54.0-4ubuntu3.1_amd64.deb ... + Unpacking libboost-thread1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Selecting previously unselected package libevent-2.0-5:amd64. + Preparing to unpack .../libevent-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-core-2.0-5:amd64. + Preparing to unpack .../libevent-core-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package libevent-pthreads-2.0-5:amd64. + Preparing to unpack .../libevent-pthreads-2.0-5_2.0.21-stable-1ubuntu1.14.04.1_amd64.deb ... + Unpacking libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Selecting previously unselected package fonts-dejavu-core. + Preparing to unpack .../fonts-dejavu-core_2.34-1ubuntu1_all.deb ... + Unpacking fonts-dejavu-core (2.34-1ubuntu1) ... + Selecting previously unselected package fontconfig-config. + Preparing to unpack .../fontconfig-config_2.11.0-0ubuntu4.2_all.deb ... + Unpacking fontconfig-config (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libfreetype6:amd64. + Preparing to unpack .../libfreetype6_2.5.2-1ubuntu2.5_amd64.deb ... + Unpacking libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Selecting previously unselected package libfontconfig1:amd64. + Preparing to unpack .../libfontconfig1_2.11.0-0ubuntu4.2_amd64.deb ... + Unpacking libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Selecting previously unselected package libjpeg-turbo8:amd64. + Preparing to unpack .../libjpeg-turbo8_1.3.0-0ubuntu2_amd64.deb ... + Unpacking libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Selecting previously unselected package libjpeg8:amd64. + Preparing to unpack .../libjpeg8_8c-2ubuntu8_amd64.deb ... + Unpacking libjpeg8:amd64 (8c-2ubuntu8) ... + Selecting previously unselected package libjbig0:amd64. + Preparing to unpack .../libjbig0_2.0-2ubuntu4.1_amd64.deb ... + Unpacking libjbig0:amd64 (2.0-2ubuntu4.1) ... + Selecting previously unselected package libtiff5:amd64. + Preparing to unpack .../libtiff5_4.0.3-7ubuntu0.4_amd64.deb ... + Unpacking libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Selecting previously unselected package libvpx1:amd64. + Preparing to unpack .../libvpx1_1.3.0-2_amd64.deb ... + Unpacking libvpx1:amd64 (1.3.0-2) ... + Selecting previously unselected package libxpm4:amd64. + Preparing to unpack .../libxpm4_1%3a3.5.10-1_amd64.deb ... + Unpacking libxpm4:amd64 (1:3.5.10-1) ... + Selecting previously unselected package libgd3:amd64. + Preparing to unpack .../libgd3_2.1.0-3ubuntu0.5_amd64.deb ... + Unpacking libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Selecting previously unselected package libperl5.18. + Preparing to unpack .../libperl5.18_5.18.2-2ubuntu1.1_amd64.deb ... + Unpacking libperl5.18 (5.18.2-2ubuntu1.1) ... + Selecting previously unselected package libsnmp-base. + Preparing to unpack .../libsnmp-base_5.7.2~dfsg-clearwater4_all.deb ... + Unpacking libsnmp-base (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package libsnmp30:amd64. + Preparing to unpack .../libsnmp30_5.7.2~dfsg-clearwater4_amd64.deb ... + Unpacking libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Selecting previously unselected package libxslt1.1:amd64. + Preparing to unpack .../libxslt1.1_1.1.28-2build1_amd64.deb ... + Unpacking libxslt1.1:amd64 (1.1.28-2build1) ... + Selecting previously unselected package geoip-database. + Preparing to unpack .../geoip-database_20140313-1_all.deb ... + Unpacking geoip-database (20140313-1) ... + Selecting previously unselected package xml-core. + Preparing to unpack .../xml-core_0.13+nmu2_all.deb ... + Unpacking xml-core (0.13+nmu2) ... + Selecting previously unselected package clearwater-log-cleanup. + Preparing to unpack .../clearwater-log-cleanup_1.0-161102.114310_all.deb ... + Unpacking clearwater-log-cleanup (1.0-161102.114310) ... + Selecting previously unselected package nginx-common. + Preparing to unpack .../nginx-common_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx-common (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx-core. + Preparing to unpack .../nginx-core_1.4.6-1ubuntu3.7_amd64.deb ... + Unpacking nginx-core (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package nginx. + Preparing to unpack .../nginx_1.4.6-1ubuntu3.7_all.deb ... + Unpacking nginx (1.4.6-1ubuntu3.7) ... + Selecting previously unselected package clearwater-nginx. + Preparing to unpack .../clearwater-nginx_1.0-161028.000011_all.deb ... + Unpacking clearwater-nginx (1.0-161028.000011) ... + Selecting previously unselected package clearwater-socket-factory. + Preparing to unpack .../clearwater-socket-factory_1.0-161102.114310_amd64.deb ... + Unpacking clearwater-socket-factory (1.0-161102.114310) ... + Selecting previously unselected package clearwater-tcp-scalability. + Preparing to unpack .../clearwater-tcp-scalability_1.0-161102.114310_all.deb ... + Unpacking clearwater-tcp-scalability (1.0-161102.114310) ... + Selecting previously unselected package sprout-libs. + Preparing to unpack .../sprout-libs_1.0-161104.171821_amd64.deb ... + Unpacking sprout-libs (1.0-161104.171821) ... + Selecting previously unselected package sprout-base. + Preparing to unpack .../sprout-base_1.0-161104.171821_amd64.deb ... + Unpacking sprout-base (1.0-161104.171821) ... + Selecting previously unselected package sprout-scscf. + Preparing to unpack .../sprout-scscf_1.0-161104.171821_amd64.deb ... + Unpacking sprout-scscf (1.0-161104.171821) ... + Selecting previously unselected package sprout-icscf. + Preparing to unpack .../sprout-icscf_1.0-161104.171821_amd64.deb ... + Unpacking sprout-icscf (1.0-161104.171821) ... + Selecting previously unselected package sprout-bgcf. + Preparing to unpack .../sprout-bgcf_1.0-161104.171821_amd64.deb ... + Unpacking sprout-bgcf (1.0-161104.171821) ... + Selecting previously unselected package sprout-mmtel-as. + Preparing to unpack .../sprout-mmtel-as_1.0-161104.171821_amd64.deb ... + Unpacking sprout-mmtel-as (1.0-161104.171821) ... + Selecting previously unselected package sprout. + Preparing to unpack .../sprout_1.0-161104.171821_amd64.deb ... + Unpacking sprout (1.0-161104.171821) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up libgeoip1:amd64 (1.6.0-1) ... + Setting up libpci3:amd64 (1:3.2.1-1ubuntu5.1) ... + Setting up libxml2:amd64 (2.9.1+dfsg1-3ubuntu4.8) ... + Setting up sgml-base (1.26+nmu4ubuntu1) ... + Setting up libboost-system1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libboost-filesystem1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libicu52:amd64 (52.1-3ubuntu0.4) ... + Setting up libboost-regex1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libboost-thread1.54.0:amd64 (1.54.0-4ubuntu3.1) ... + Setting up libevent-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-core-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up libevent-pthreads-2.0-5:amd64 (2.0.21-stable-1ubuntu1.14.04.1) ... + Setting up fonts-dejavu-core (2.34-1ubuntu1) ... + Setting up fontconfig-config (2.11.0-0ubuntu4.2) ... + Setting up libfreetype6:amd64 (2.5.2-1ubuntu2.5) ... + Setting up libfontconfig1:amd64 (2.11.0-0ubuntu4.2) ... + Setting up libjpeg-turbo8:amd64 (1.3.0-0ubuntu2) ... + Setting up libjpeg8:amd64 (8c-2ubuntu8) ... + Setting up libjbig0:amd64 (2.0-2ubuntu4.1) ... + Setting up libtiff5:amd64 (4.0.3-7ubuntu0.4) ... + Setting up libvpx1:amd64 (1.3.0-2) ... + Setting up libxpm4:amd64 (1:3.5.10-1) ... + Setting up libgd3:amd64 (2.1.0-3ubuntu0.5) ... + Setting up libperl5.18 (5.18.2-2ubuntu1.1) ... + Setting up libsnmp-base (5.7.2~dfsg-clearwater4) ... + Setting up libsnmp30:amd64 (5.7.2~dfsg-clearwater4) ... + Setting up libxslt1.1:amd64 (1.1.28-2build1) ... + Setting up geoip-database (20140313-1) ... + Setting up xml-core (0.13+nmu2) ... + Setting up clearwater-log-cleanup (1.0-161102.114310) ... + Setting up nginx-common (1.4.6-1ubuntu3.7) ... + Setting up clearwater-socket-factory (1.0-161102.114310) ... + Setting up clearwater-tcp-scalability (1.0-161102.114310) ... + Setting up sprout-libs (1.0-161104.171821) ... + Processing triggers for ureadahead (0.100.0-16) ... + Setting up nginx-core (1.4.6-1ubuntu3.7) ... + invoke-rc.d: policy-rc.d denied execution of start. + Setting up nginx (1.4.6-1ubuntu3.7) ... + Setting up clearwater-nginx (1.0-161028.000011) ... + Testing nginx configuration... + Site ping has been enabled. Run /etc/init.d/nginx reload to apply the changes. + nginx: the configuration file /etc/nginx/nginx.conf syntax is ok + nginx: configuration file /etc/nginx/nginx.conf test is successful + Generating a 2048 bit RSA private key + ...........................+++ + ..........................+++ + writing new private key to 'nginx.key' + ----- + Signature ok + subject=/C=UK/ST=Unknown state/L=Unknown locality/O=Unknown organization/OU=Unknown organizational unit/CN=unknown.example/emailAddress=Unknown + Getting Private key + Adding 'diversion of /etc/nginx/nginx.conf to /etc/nginx/nginx.conf.clearwater-orig by clearwater-nginx' + Setting up sprout-base (1.0-161104.171821) ... + * Restarting clearwater-infrastructure clearwater-infrastructure + mv: cannot move 鈥▒/tmp/hosts.784鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + * Restarting DNS forwarder and DHCP server dnsmasq + + dnsmasq: setting capabilities failed: Operation not permitted + ...fail! + * Stopping enhanced syslogd rsyslogd + ...done. + * Starting enhanced syslogd rsyslogd + ...done. + hostname: you must be root to change the host name + mv: cannot move 鈥▒/tmp/hosts.844鈥▒ to 鈥▒/etc/hosts鈥▒: Device or resource busy + Configuring monit for only localhost access + ...done. + Processing triggers for ureadahead (0.100.0-16) ... + Setting up sprout-bgcf (1.0-161104.171821) ... + Setting up sprout-mmtel-as (1.0-161104.171821) ... + Setting up sprout-scscf (1.0-161104.171821) ... + Setting up sprout-icscf (1.0-161104.171821) ... + Setting up sprout (1.0-161104.171821) ... + Processing triggers for libc-bin (2.19-0ubuntu6.9) ... + Processing triggers for sgml-base (1.26+nmu4ubuntu1) ... + ---> 61bb42090ea1 + Removing intermediate container b9ae61eb4e92 + Step 6 : ADD plugins/* /usr/share/clearwater/clearwater-cluster-manager/plugins/ + ---> 8351cb90eb8b + Removing intermediate container 18ae3869cedc + Step 7 : ADD reload_memcached_users /usr/share/clearwater/bin/reload_memcached_users + ---> 9df0746cb954 + Removing intermediate container 3be50b3e9f4f + Step 8 : EXPOSE 5052 5054 + ---> Running in b7d01534f817 + ---> f02626e19a4c + Removing intermediate container b7d01534f817 + Successfully built f02626e19a4c diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-deployment.yaml new file mode 100755 index 0000000..df712d1 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: bono + name: bono + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: bono + template: + metadata: + labels: + service: bono + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_bono # docker.io/tangfeixiong/clearwater-bono:1611280555.gitrev-3e9722f + name: bono + ports: + - containerPort: 22 + - containerPort: 3478 + hostPort: 3478 # Currently we use docker proxy, maybe pod hostnetwork + - containerPort: 5060 + hostPort: 5060 + - containerPort: 5062 + hostPort: 5062 + - containerPort: 5060 + protocol: UDP + - containerPort: 5062 + protocol: UDP + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "3478 5060 5062"] + # initialDelaySeconds: 30 + # readinessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "3478 5060 5062"] + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-service-dockerproxy.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-service-dockerproxy.yaml new file mode 100755 index 0000000..6f329c4 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-service-dockerproxy.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: bono + namespace: default + labels: + proxy: docker +spec: + clusterIP: None + ports: + - name: "stun3478" + port: 3478 + - name: "sip5060" + port: 5060 + - name: "sip5062" + port: 5062 + - name: "sip5060udp" + port: 5060 + protocol: UDP + - name: "sip5062udp" + port: 5062 + protocol: UDP + selector: + service: bono \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-service.yaml new file mode 100755 index 0000000..39d9986 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/bono-service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: bono + namespace: default + labels: + proxy: docker +spec: + clusterIP: None # Because we use docker proxy or pod hostnetwork + ports: + - name: "stun3478" + port: 3478 + - name: "sip5060" + port: 5060 + - name: "sip5062" + port: 5062 + - name: "sip5060udp" + port: 5060 + protocol: UDP + - name: "sip5062udp" + port: 5062 + protocol: UDP + selector: + service: bono \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/cassandra-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/cassandra-deployment.yaml new file mode 100755 index 0000000..8f03d4a --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/cassandra-deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: cassandra + name: cassandra + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: cassandra + template: + metadata: + labels: + service: cassandra + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_cassandra # docker.io/tangfeixiong/clearwater-cassandra:1611280555.gitrev-3e9722f + name: cassandra + ports: + - containerPort: 22 + - containerPort: 7001 + - containerPort: 9042 + - containerPort: 9160 + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "7000 9042 9160"] + # initialDelaySeconds: 30 + # readinessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "7000 9042 9160"] + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/cassandra-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/cassandra-service.yaml new file mode 100755 index 0000000..0510f34 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/cassandra-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: cassandra + namespace: default +spec: + clusterIP: None + ports: + - name: "7001" + port: 7001 + - name: "7000" + port: 7000 + - name: "9042" + port: 9042 + - name: "9160" + port: 9160 + selector: + service: cassandra \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/chronos-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/chronos-deployment.yaml new file mode 100755 index 0000000..3d5b980 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/chronos-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: chronos + name: chronos +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: chronos + template: + metadata: + labels: + service: chronos + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_chronos # docker.io/tangfeixiong/clearwater-chronos:1611280555.gitrev-3e9722f + name: chronos + ports: + - containerPort: 22 + - containerPort: 7253 + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # tcpSocket: + # port: 7253 + # initialDelaySeconds: 30 + # readinessProbe: + # tcpSocket: + # port: 7253 + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/chronos-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/chronos-service.yaml new file mode 100755 index 0000000..22533f4 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/chronos-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: chronos + namespace: default +spec: + clusterIP: None + ports: + - name: "7253" + port: 7253 + selector: + service: chronos \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-deployment.yaml new file mode 100755 index 0000000..4ba0c54 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: ellis + name: ellis + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: ellis + template: + metadata: + labels: + service: ellis + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_ellis # docker.io/tangfeixiong/clearwater-ellis:1611280555.gitrev-3e9722f + name: ellis + ports: + - containerPort: 22 + - containerPort: 80 + hostPort: 38080 # This can be directly proxy by docker, also maybe pod hostnetwork + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # tcpSocket: + # port: 80 + # initialDelaySeconds: 30 + # readinessProbe: + # tcpSocket: + # port: 80 + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-service-dockerproxy.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-service-dockerproxy.yaml new file mode 100755 index 0000000..7e192ac --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-service-dockerproxy.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: ellis-dockerproxy + namespace: default +spec: + clusterIP: None + ports: + - name: account + port: 80 + targetPort: 80 + selector: + service: ellis \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-service.yaml new file mode 100755 index 0000000..256eead --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ellis-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: ellis + namespace: default +spec: + ports: + - name: "80" + port: 80 + nodePort: 32180 + targetPort: 80 + selector: + service: ellis + type: NodePort \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/etcd-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/etcd-deployment.yaml new file mode 100755 index 0000000..e3e391f --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/etcd-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: etcd + name: etcd + namespace: default +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + service: etcd + template: + metadata: + labels: + service: etcd + spec: + containers: + - args: + - -name + - etcd0 + - -advertise-client-urls + - http://etcd:2379,http://etcd:4001 + - -listen-client-urls + - http://0.0.0.0:2379,http://0.0.0.0:4001 + - -initial-advertise-peer-urls + - http://etcd:2380 + - -listen-peer-urls + - http://0.0.0.0:2380 + - -initial-cluster + - etcd0=http://etcd:2380 + - -initial-cluster-state + - new + image: quay.io/coreos/etcd:v2.2.5 + name: etcd + ports: + - containerPort: 2379 + - containerPort: 2380 + - containerPort: 4001 + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/etcd-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/etcd-service.yaml new file mode 100755 index 0000000..85147b2 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/etcd-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: etcd + namespace: default +spec: + clusterIP: None + ports: + - name: client + port: 2379 + - name: "4001" + port: 4001 + - name: peer + port: 2380 + selector: + service: etcd \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/homer-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homer-deployment.yaml new file mode 100755 index 0000000..8f443c7 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homer-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: homer + name: homer + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: homer + template: + metadata: + labels: + service: homer + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_homer # docker.io/tangfeixiong/clearwater-homer:1611280555.gitrev-3e9722f + name: homer + ports: + - containerPort: 22 + - containerPort: 7888 + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # tcpSocket: + # port: 7888 + # initialDelaySeconds: 30 + # readinessProbe: + # tcpSocket: + # port: 7888 + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/homer-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homer-service.yaml new file mode 100755 index 0000000..ef1fab0 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homer-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: homer + namespace: default +spec: + clusterIP: None + ports: + - name: "7888" + port: 7888 + selector: + service: homer \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/homestead-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homestead-deployment.yaml new file mode 100755 index 0000000..376b9ef --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homestead-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: homestead + name: homestead + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: homestead + template: + metadata: + labels: + service: homestead + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_homestead # docker.io/tangfeixiong/clearwater-homestead:1611280555.gitrev-3e9722f + name: homestead + ports: + - containerPort: 22 + - containerPort: 8888 + - containerPort: 8889 + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "8888 8889"] + # initialDelaySeconds: 30 + # readinessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "8888 8889"] + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/homestead-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homestead-service.yaml new file mode 100755 index 0000000..99fcd34 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/homestead-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: homestead + namespace: default +spec: + clusterIP: None + ports: + - name: "8888" + port: 8888 + - name: "8889" + port: 8889 + selector: + service: homestead \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/memcached-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/memcached-deployment.yaml new file mode 100755 index 0000000..b8dad3e --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/memcached-deployment.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: memcached + name: memcached + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: memcached + template: + metadata: + labels: + service: memcached + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_memcached # docker.io/tangfeixiong/clearwater-memcached:1611280555.gitrev-3e9722f + name: memcached + ports: + - containerPort: 22 + - containerPort: 11211 + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # tcpSocket: + # port: 11211 + # initialDelaySeconds: 30 + # readinessProbe: + # tcpSocket: + # port: 11211 + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/memcached-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/memcached-service.yaml new file mode 100755 index 0000000..f210b52 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/memcached-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: memcached + namespace: default +spec: + clusterIP: None + ports: + - name: "11211" + port: 11211 + selector: + service: memcached \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/ralf-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ralf-deployment.yaml new file mode 100755 index 0000000..4f1880c --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ralf-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +kind: Service +metadata: + name: memcached + namespace: default +spec: + clusterIP: None + ports: + - name: "11211" + port: 11211 + selector: + service: memcached +[tangfx@localhost k8s-manifests]$ cat ralf-deployment.yaml +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: ralf + name: ralf + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: ralf + template: + metadata: + labels: + service: ralf + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_ralf # docker.io/tangfeixiong/clearwater-ralf:1611280555.gitrev-3e9722f + name: ralf + ports: + - containerPort: 22 + - containerPort: 10888 + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # tcpSocket: + # port: 10888 + # initialDelaySeconds: 30 + # readinessProbe: + # tcpSocket: + # port: 10888 + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/ralf-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ralf-service.yaml new file mode 100755 index 0000000..37fd8d0 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/ralf-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: ralf + namespace: default +spec: + clusterIP: None + ports: + - name: "10888" + port: 10888 + selector: + service: ralf \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/sprout-deployment.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/sprout-deployment.yaml new file mode 100755 index 0000000..20ea585 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/sprout-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + service: sprout + name: sprout + namespace: default +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + service: sprout + template: + metadata: + labels: + service: sprout + spec: + containers: + - image: 10.64.33.81:5000/admin/clearwater_sprout # docker.io/tangfeixiong/clearwater-sprout:1611280555.gitrev-3e9722f + name: sprout + ports: + - containerPort: 22 + - containerPort: 5052 + - containerPort: 5054 + env: + # - name: ETCD_PROXY + # value: default=http://10.64.33.81:2380 + # - name: ZONE + # value: default.svc.cluster.local + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # livenessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "5052 5054"] + # initialDelaySeconds: 30 + # readinessProbe: + # exec: + # command: ["/bin/bash", "/usr/share/kubernetes/liveness.sh", "5052 5054"] + restartPolicy: Always \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/k8s-manifests/sprout-service.yaml b/hack/kubernetes-clearwater/clearwater/k8s-manifests/sprout-service.yaml new file mode 100755 index 0000000..e3e6a21 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/k8s-manifests/sprout-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: sprout + namespace: default +spec: + clusterIP: None + ports: + - name: icscf + port: 5052 + - name: scscf + port: 5054 + selector: + service: sprout \ No newline at end of file diff --git a/hack/kubernetes-clearwater/clearwater/local_repo.md b/hack/kubernetes-clearwater/clearwater/local_repo.md new file mode 100755 index 0000000..3d86df2 --- /dev/null +++ b/hack/kubernetes-clearwater/clearwater/local_repo.md @@ -0,0 +1,42 @@ + +### A `cw-ngv` local repo + +Install and run tool + + ubuntu@ubuntu-14-04-3-server-amd64: ~$ sudo apt-get install apt-mirror + + ubuntu@ubuntu-14-04-3-server-amd64: ~$ sudo vi /etc/apt/mirror.list + + deb http://repo.cw-ngv.com/stable binary + + clean http://repo.cw-ngv.com/stable + + ubuntu@ubuntu-14-04-3-server-amd64: ~$ sudo apt-mirror + + ubuntu@ubuntu-14-04-3-server-amd64: ~$ ls /var/spool/apt-mirror/mirror/repo.cw-ngv.com/stable/binary/ + + ubuntu@ubuntu-14-04-3-server-amd64: ~$ rsync -avz -e ssh --delete /var/spool/apt-mirror/mirror/repo.cw-ngv.com tangf@192.168.0.250:/cygdrive/f/16-mirror/apt-mirror/mirror + +Cache repo gpg + + tangf@DESKTOP-H68OQDV /cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker/base + $ curl -L http://repo.cw-ngv.com/repo_key -o /cygdrive/f/16-mirror/apt-mirror/gpg-key/repo.cw-ngv.com/repo_key + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 100 3149 100 3149 0 0 3002 0 0:00:01 0:00:01 --:--:-- 4928 + +Run `gofileserver` + + tangf@DESKTOP-H68OQDV /cygdrive/f/16-mirror/apt-mirror/trusty + $ gofileserver.exe + Listening at :48080 + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ curl http://10.64.33.1:48080/ + 
+    gpg-key/
+    mirror/
+
+ + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ curl -sL http://10.64.33.1:48080/gpg-key/http%253A%252F%252Frepo.cw-ngv.com/repo_key -o- diff --git a/hack/kubernetes-clearwater/docker-installation.md b/hack/kubernetes-clearwater/docker-installation.md new file mode 100755 index 0000000..f173db0 --- /dev/null +++ b/hack/kubernetes-clearwater/docker-installation.md @@ -0,0 +1,209 @@ +Instruction +============ + +After installed CentOS box + + tangf@DESKTOP-H68OQDV /cygdrive/g/atlas.hashicorp.com/centos/boxes/7 + $ vagrant ssh + Last login: Mon Nov 7 11:36:53 2016 from 10.0.2.2 + + [vagrant@localhost ~]$ sudo systemctl list-unit-files docker.service + UNIT FILE STATE + + 0 unit files listed. + +CentOS7.2的docker RPM packages + + [vagrant@localhost ~]$ sudo yum list docker + Loaded plugins: fastestmirror + base | 3.6 kB 00:00 + extras | 3.4 kB 00:00 + updates | 3.4 kB 00:00 + (1/4): base/7/x86_64/group_gz | 155 kB 00:00 + (2/4): extras/7/x86_64/primary_db | 166 kB 00:02 + (3/4): updates/7/x86_64/primary_db | 9.1 MB 00:16 + (4/4): base/7/x86_64/primary_db | 5.3 MB 00:36 + Determining fastest mirrors + * base: mirror.lzu.edu.cn + * extras: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + Available Packages + docker.x86_64 1.10.3-46.el7.centos.14 extras + +POC +---- + +[Official site](http://mirror.centos.org/centos/7/paas/x86_64/openshift-origin/) + +[Mirror list](https://www.centos.org/download/mirrors/) + +[阿里云](http://mirrors.aliyun.com/docker-engine/) + +For rsync => rsync://mirrors.yun-idc.com/centos/ + +* Sync CentOS for example Extras repo + +[脚本](./scripts)参见scripts目录 + + tangf@DESKTOP-H68OQDV /cygdrive/g/2015-12-19-repository/99-mirror/centos + $ ./mirror-by-rsync.sh + +* A windows hosted scratch file server （written by Golang） + +[代码](./gofileserver)参见gofileserver目录 + + tangf@DESKTOP-H68OQDV /cygdrive/g/2015-12-19-repository/99-mirror/centos/rsync%3A%2F%2Fmirrors.yun-idc.com%2Fcentos%2F7 + $ gofileserver.exe + Listening at :48080 + +* Configure into CentOS box + +设置Extras repo + + [vagrant@localhost ~]$ sudo yum-config-manager --disable extras + Loaded plugins: fastestmirror + ================================= repo: extras ================================= + [extras] + async = True + bandwidth = 0 + base_persistdir = /var/lib/yum/repos/x86_64/7 + baseurl = + cache = 0 + cachedir = /var/cache/yum/x86_64/7/extras + check_config_file_age = True + cost = 1000 + deltarpm_metadata_percentage = 100 + deltarpm_percentage = + enabled = 0 + enablegroups = True + exclude = + failovermethod = priority + gpgcadir = /var/lib/yum/repos/x86_64/7/extras/gpgcadir + gpgcakey = + gpgcheck = True + gpgdir = /var/lib/yum/repos/x86_64/7/extras/gpgdir + gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + hdrdir = /var/cache/yum/x86_64/7/extras/headers + http_caching = all + includepkgs = + ip_resolve = + keepalive = True + keepcache = False + mddownloadpolicy = sqlite + mdpolicy = group:small + mediaid = + metadata_expire = 21600 + metadata_expire_filter = read-only:present + metalink = + minrate = 0 + mirrorlist = http://mirrorlist.centos.org/?release=7&arch=x86_64&repo=extras&infra=vag + mirrorlist_expire = 86400 + name = CentOS-7 - Extras + old_base_cache_dir = + password = + persistdir = /var/lib/yum/repos/x86_64/7/extras + pkgdir = /var/cache/yum/x86_64/7/extras/packages + proxy = False + proxy_dict = + proxy_password = + proxy_username = + repo_gpgcheck = False + retries = 10 + skip_if_unavailable = False + ssl_check_cert_permissions = True + sslcacert = + sslclientcert = + sslclientkey = + sslverify = True + throttle = 0 + timeout = 30.0 + ui_id = extras/7/x86_64 + ui_repoid_vars = releasever, + basearch + username = + + [vagrant@localhost ~]$ sudo tee /etc/yum.repos.d/local-mirror-extras.repo <<- 'EOF' + > [extras-mirror] + > name=CentOS-7 - Extras + > baseurl=http://192.168.1.100:48080/extras/x86_64 + > enabled=1 + > gpgcheck=1 + > gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + > EOF + + [extras-mirror] + name=CentOS-7 - Extras + baseurl=http://192.168.1.100:48080/extras/x86_64 + enabled=1 + gpgcheck=1 + gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + + +Experimental ops + + [vagrant@localhost ~]$ sudo yum repolist Loaded plugins: fastestmirror + base | 3.6 kB 00:00 + centos-openshift-origin | 2.9 kB 00:00 + extras | 3.4 kB 00:00 + updates | 3.4 kB 00:00 + centos-openshift-origin/primary_db | 87 kB 00:01 + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * extras: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + repo id repo name status + base/7/x86_64 CentOS-7 - Base 9,007 + centos-openshift-origin CentOS OpenShift Origin 107 + extras/7/x86_64 CentOS-7 - Extras 393 + updates/7/x86_64 CentOS-7 - Updates 2,560 + repolist: 12,067 + + [vagrant@localhost ~]$ sudo yum --disablerepo=extras,centos-openshift-origin --enablerepo=extras-mirror,centos-openshift-origin-mirror repolist + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + repo id repo name status + base/7/x86_64 CentOS-7 - Base 9,007 + !centos-openshift-origin-mirror CentOS Openshift Origin 107 + !extras-mirror CentOS-7 - Extras 393 + updates/7/x86_64 CentOS-7 - Updates 2,560 + repolist: 12,067 + +* Install docker + +参考[docker官方文档](https://docs.docker.com/engine/installation/linux/centos/) + + [vagrant@localhost ~]$ sudo systemctl enable docker.service + + [vagrant@localhost ~]$ sudo systemctl start docker.service + +Validation + + [vagrant@localhost ~]$ sudo systemctl -l status docker.service + 鈼▒ docker.service - Docker Application Container Engine + Loaded: loaded (/usr/lib/systemd/system/docker.service; enabled; vendor preset: disabled) + Drop-In: /usr/lib/systemd/system/docker.service.d + 鈹斺攢flannel.conf + Active: active (running) since Sat 2016-11-12 22:43:09 UTC; 10s ago + Docs: http://docs.docker.com + Main PID: 13547 (docker-current) + CGroup: /system.slice/docker.service + 鈹斺攢13547 /usr/bin/docker-current daemon --exec-opt native.cgroupdriver=systemd --selinux-enabled --log-driver=journald --bip=10.121.24.1/23 --ip-masq=true --mtu=1472 + + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.824941328Z" level=warning msg="devmapper: Base device already exists and has filesystem xfs on it. User specified filesystem will be ignored." + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.847090355Z" level=info msg="[graphdriver] using prior storage driver \"devicemapper\"" + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.849853203Z" level=info msg="Graph migration to content-addressability took 0.00 seconds" + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.856844053Z" level=info msg="Firewalld running: false" + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.953737295Z" level=info msg="Loading containers: start." + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.953787544Z" level=info msg="Loading containers: done." + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.953795950Z" level=info msg="Daemon has completed initialization" + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.953806548Z" level=info msg="Docker daemon" commit=cb079f6-unsupported execdriver=native-0.2 graphdriver=devicemapper version=1.10.3 + Nov 12 22:43:09 localhost.localdomain docker-current[13547]: time="2016-11-12T22:43:09.957962358Z" level=info msg="API listen on /var/run/docker.sock" + Nov 12 22:43:09 localhost.localdomain systemd[1]: Started Docker Application Container Engine. + + [vagrant@localhost ~]$ docker network ls + NETWORK ID NAME DRIVER + 6946ae42963c bridge bridge + e1f79a5b1ddb none null + f6b9fe847c0a host host diff --git a/hack/kubernetes-clearwater/etcd-installation.md b/hack/kubernetes-clearwater/etcd-installation.md new file mode 100755 index 0000000..ae87b54 --- /dev/null +++ b/hack/kubernetes-clearwater/etcd-installation.md @@ -0,0 +1,286 @@ + +POC +---- + +About etcd2 package + + [vagrant@localhost ~]$ sudo yum -v list etcd + Loading "fastestmirror" plugin + Config time: 0.005 + Yum version: 3.4.3 + rpmdb time: 0.000 + Setting up Package Sacks + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * extras: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + pkgsack time: 0.007 + Available Packages + etcd.x86_64 2.3.7-4.el7 extras + + [vagrant@localhost ~]$ sudo yum deplist etcd + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * extras: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + package: etcd.x86_64 2.3.7-4.el7 + dependency: /bin/sh + provider: bash.x86_64 4.2.46-20.el7_2 + dependency: shadow-utils + provider: shadow-utils.x86_64 2:4.1.5.1-18.el7 + dependency: systemd + provider: systemd.x86_64 219-19.el7_2.13 + +* Install from local repo + +Using *gofileserver* + + tangf@DESKTOP-H68OQDV /cygdrive/g/2015-12-19-repository/99-mirror/centos/rsync%3A%2F%2Fmirrors.yun-idc.com%2Fcentos%2F7 + $ cd /cygdrive/i/mirror/centos/rsync%3A%2F%2Fmirrors.yun-idc.com%2Fcentos%2F7/ + + tangf@DESKTOP-H68OQDV /cygdrive/i/mirror/centos/rsync%3A%2F%2Fmirrors.yun-idc.com%2Fcentos%2F7 + $ gofileserver.exe + Listening at :48080 + + [vagrant@localhost ~]$ sudo yum --disablerepo=extras --enablerepo=extras-mirror install -y etcd + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + Resolving Dependencies + --> Running transaction check + ---> Package etcd.x86_64 0:2.3.7-4.el7 will be installed + --> Finished Dependency Resolution + + Dependencies Resolved + + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Installing: + etcd x86_64 2.3.7-4.el7 extras-mirror 6.5 M + + Transaction Summary + ================================================================================ + Install 1 Package + + Total download size: 6.5 M + Installed size: 31 M + Downloading packages: + etcd-2.3.7-4.el7.x86_64.rpm | 6.5 MB 00:06 + Running transaction check + Running transaction test + Transaction test succeeded + Running transaction + Installing : etcd-2.3.7-4.el7.x86_64 1/1 + Verifying : etcd-2.3.7-4.el7.x86_64 1/1 + + Installed: + etcd.x86_64 0:2.3.7-4.el7 + + Complete! + +Deep dive into etcd2 + + [vagrant@localhost ~]$ sudo systemctl list-unit-files etcd.service + UNIT FILE STATE + etcd.service disabled + + 1 unit files listed. + + [vagrant@localhost ~]$ cat /usr/lib/systemd/system/etcd.service + [Unit] + Description=Etcd Server + After=network.target + After=network-online.target + Wants=network-online.target + + [Service] + Type=notify + WorkingDirectory=/var/lib/etcd/ + EnvironmentFile=-/etc/etcd/etcd.conf + User=etcd + # set GOMAXPROCS to number of processors + ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\"" + Restart=on-failure + LimitNOFILE=65536 + + [Install] + WantedBy=multi-user.target + + [vagrant@localhost ~]$ ls -ld /var/lib/etcd + drwxr-xr-x. 2 etcd etcd 6 Sep 16 13:18 /var/lib/etcd + + [vagrant@localhost ~]$ cat /etc/passwd | grep etcd + etcd:x:994:990:etcd user:/var/lib/etcd:/sbin/nologin + + [vagrant@localhost ~]$ sudo vi /etc/etcd/etcd.conf + # [member] + ETCD_NAME=default + ETCD_DATA_DIR="/var/lib/etcd/default.etcd" + #ETCD_WAL_DIR="" + #ETCD_SNAPSHOT_COUNT="10000" + #ETCD_HEARTBEAT_INTERVAL="100" + #ETCD_ELECTION_TIMEOUT="1000" + #ETCD_LISTEN_PEER_URLS="http://localhost:2380" + #ETCD_LISTEN_CLIENT_URLS="http://localhost:2379" + ETCD_LISTEN_CLIENT_URLS="http://10.64.33.81:2379" + #ETCD_MAX_SNAPSHOTS="5" + #ETCD_MAX_WALS="5" + #ETCD_CORS="" + # + #[cluster] + #ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380" + # if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." + #ETCD_INITIAL_CLUSTER="default=http://localhost:2380" + #ETCD_INITIAL_CLUSTER_STATE="new" + #ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" + #ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://0.0.0.0:2379" + #ETCD_DISCOVERY="" + #ETCD_DISCOVERY_SRV="" + #ETCD_DISCOVERY_FALLBACK="proxy" + #ETCD_DISCOVERY_PROXY="" + #ETCD_STRICT_RECONFIG_CHECK="false" + # + #[proxy] + #ETCD_PROXY="off" + #ETCD_PROXY_FAILURE_WAIT="5000" + #ETCD_PROXY_REFRESH_INTERVAL="30000" + #ETCD_PROXY_DIAL_TIMEOUT="1000" + #ETCD_PROXY_WRITE_TIMEOUT="5000" + #ETCD_PROXY_READ_TIMEOUT="0" + # + #[security] + #ETCD_CERT_FILE="" + #ETCD_KEY_FILE="" + #ETCD_CLIENT_CERT_AUTH="false" + #ETCD_TRUSTED_CA_FILE="" + #ETCD_PEER_CERT_FILE="" + #ETCD_PEER_KEY_FILE="" + #ETCD_PEER_CLIENT_CERT_AUTH="false" + #ETCD_PEER_TRUSTED_CA_FILE="" + # + #[logging] + #ETCD_DEBUG="false" + # examples for -log-package-levels etcdserver=WARNING,security=DEBUG + #ETCD_LOG_PACKAGE_LEVELS="" + # + #[profiling] + #ETCD_ENABLE_PPROF="false" + + [vagrant@localhost ~]$ sudo systemctl enable etcd.service + Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service. + + [vagrant@localhost ~]$ ls -l /etc/systemd/system/multi-user.target.wants/etcd.service + lrwxrwxrwx. 1 root root 36 Nov 10 20:20 /etc/systemd/system/multi-user.target.wants/etcd.service -> /usr/lib/systemd/system/etcd.service + + [vagrant@localhost ~]$ sudo systemctl start etcd.service + + [vagrant@localhost ~]$ sudo systemctl -l status etcd.service + 鈼▒ etcd.service - Etcd Server + Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled) + Active: active (running) since Thu 2016-11-10 20:25:15 UTC; 52s ago + Main PID: 4473 (etcd) + CGroup: /system.slice/etcd.service + 鈹斺攢4473 /usr/bin/etcd --name=default --data-dir=/var/lib/etcd/default.etcd --listen-client-urls=http://0.0.0.0:2379 + + Nov 10 20:25:15 localhost.localdomain etcd[4473]: starting server... [version: 2.3.7, cluster version: to_be_decided] + Nov 10 20:25:15 localhost.localdomain systemd[1]: Started Etcd Server. + Nov 10 20:25:15 localhost.localdomain etcd[4473]: added local member ce2a822cea30bfca [http://localhost:2380 http://localhost:7001] to cluster 7e27652122e8b2ae + Nov 10 20:25:15 localhost.localdomain etcd[4473]: set the initial cluster version to 2.3 + Nov 10 20:25:17 localhost.localdomain etcd[4473]: ce2a822cea30bfca is starting a new election at term 2 + Nov 10 20:25:17 localhost.localdomain etcd[4473]: ce2a822cea30bfca became candidate at term 3 + Nov 10 20:25:17 localhost.localdomain etcd[4473]: ce2a822cea30bfca received vote from ce2a822cea30bfca at term 3 + Nov 10 20:25:17 localhost.localdomain etcd[4473]: ce2a822cea30bfca became leader at term 3 + Nov 10 20:25:17 localhost.localdomain etcd[4473]: raft.node: ce2a822cea30bfca elected leader ce2a822cea30bfca at term 3 + Nov 10 20:25:17 localhost.localdomain etcd[4473]: published {Name:default ClientURLs:[http://10.64.33.81:2379]} to cluster 7e27652122e8b2ae + + [vagrant@localhost ~]$ sudo journalctl --no-pager --no-tail -u etcd.service + + [vagrant@localhost ~]$ curl http://127.0.0.1:2379/version + {"etcdserver":"2.3.7","etcdcluster":"2.3.0"} + + [vagrant@localhost ~]$ which etcdctl + /usr/bin/etcdctl + + [vagrant@localhost ~]$ etcdctl member list + ce2a822cea30bfca: name=default peerURLs=http://localhost:2380,http://localhost:7001 clientURLs=http://10.64.33.81:2379 isLeader=true + + [vagrant@localhost ~]$ etcdctl cluster-health + member ce2a822cea30bfca is healthy: got healthy result from http://10.64.33.81:2379 + cluster is healthy + +For cluster updation + + [vagrant@master81 ~]$ etcdctl member list + ce2a822cea30bfca: name=default peerURLs=http://localhost:2380,http://localhost:7001 clientURLs=http://10.64.33.81:2379 isLeader=true + + [vagrant@master81 ~]$ etcdctl member update ce2a822cea30bfca http://10.64.33.81:2380 + + [vagrant@master81 ~]$ etcdctl member list + ce2a822cea30bfca: name=default peerURLs=http://10.64.33.81:2380 clientURLs=http://10.64.33.81:2379 isLeader=true + + [vagrant@master81 ~]$ etcdctl ls + /clearwater + /coreos.com + /registry + + [vagrant@master81 ~]$ cat /etc/etcd/etcd.conf + [member] + ETCD_NAME=default + ETCD_DATA_DIR="/var/lib/etcd/default.etcd" + #ETCD_WAL_DIR="" + #ETCD_SNAPSHOT_COUNT="10000" + #ETCD_HEARTBEAT_INTERVAL="100" + #ETCD_ELECTION_TIMEOUT="1000" + #ETCD_LISTEN_PEER_URLS="http://localhost:2380" + ETCD_LISTEN_PEER_URLS="http://10.64.33.81:2380" + #ETCD_LISTEN_CLIENT_URLS="http://localhost:2379" + ETCD_LISTEN_CLIENT_URLS="http://10.64.33.81:2379,http://localhost:2379" + #ETCD_MAX_SNAPSHOTS="5" + #ETCD_MAX_WALS="5" + #ETCD_CORS="" + # + [cluster] + #ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.64.33.81:2380" + # if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." + #ETCD_INITIAL_CLUSTER="default=http://localhost:2380" + ETCD_INITIAL_CLUSTER="default=http://10.64.33.81:2380" + ETCD_INITIAL_CLUSTER_STATE="new" + #ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" + #ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://10.64.33.81:2379" + #ETCD_DISCOVERY="" + #ETCD_DISCOVERY_SRV="" + #ETCD_DISCOVERY_FALLBACK="proxy" + #ETCD_DISCOVERY_PROXY="" + #ETCD_STRICT_RECONFIG_CHECK="false" + # + #[proxy] + #ETCD_PROXY="off" + #ETCD_PROXY_FAILURE_WAIT="5000" + #ETCD_PROXY_REFRESH_INTERVAL="30000" + #ETCD_PROXY_DIAL_TIMEOUT="1000" + #ETCD_PROXY_WRITE_TIMEOUT="5000" + #ETCD_PROXY_READ_TIMEOUT="0" + # + #[security] + #ETCD_CERT_FILE="" + #ETCD_KEY_FILE="" + #ETCD_CLIENT_CERT_AUTH="false" + #ETCD_TRUSTED_CA_FILE="" + #ETCD_PEER_CERT_FILE="" + #ETCD_PEER_KEY_FILE="" + #ETCD_PEER_CLIENT_CERT_AUTH="false" + #ETCD_PEER_TRUSTED_CA_FILE="" + # + #[logging] + #ETCD_DEBUG="false" + # examples for -log-package-levels etcdserver=WARNING,security=DEBUG + #ETCD_LOG_PACKAGE_LEVELS="" + # + #[profiling] + #ETCD_ENABLE_PPROF="false" diff --git a/hack/kubernetes-clearwater/flanneld-installation.md b/hack/kubernetes-clearwater/flanneld-installation.md new file mode 100755 index 0000000..31014cc --- /dev/null +++ b/hack/kubernetes-clearwater/flanneld-installation.md @@ -0,0 +1,425 @@ +Instruction +------------- + +Packages + + [vagrant@localhost ~]$ yum list flannel + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * base: mirrors.cqu.edu.cn + * extras: mirrors.163.com + * updates: mirrors.163.com + Available Packages + flannel.x86_64 0.5.3-9.el7 extras + +* Prerequisites + +Firewall + + [tangfx@localhost sysconfig]$ sudo systemctl stop firewalld.service + + [tangfx@localhost sysconfig]$ sudo systemctl disable firewalld.service + +* Install from local repo + +Start gofileserver + + tangf@DESKTOP-H68OQDV /cygdrive/g/2015-12-19-repository/99-mirror/centos/rsync%3A%2F%2Fmirrors.yun-idc.com%2Fcentos%2F7 + $ gofileserver.exe + Listening at :48080 + +Install package + + [vagrant@localhost ~]$ sudo yum --disablerepo=extras --enablerepo=extras-mirror install flannel + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + Resolving Dependencies + --> Running transaction check + ---> Package flannel.x86_64 0:0.5.3-9.el7 will be installed + --> Finished Dependency Resolution + + Dependencies Resolved + + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Installing: + flannel x86_64 0.5.3-9.el7 extras-mirror 1.7 M + + Transaction Summary + ================================================================================ + Install 1 Package + + [vagrant@localhost ~]$ sudo yum --disablerepo=\* --enablerepo=c7-media install bridge-utils + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * c7-media: + Resolving Dependencies + --> Running transaction check + ---> Package bridge-utils.x86_64 0:1.5-9.el7 will be installed + --> Finished Dependency Resolution + + Dependencies Resolved + + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Installing: + bridge-utils x86_64 1.5-9.el7 c7-media 32 k + + Transaction Summary + ================================================================================ + Install 1 Package + + [vagrant@localhost ~]$ sudo brctl show + +* Containers networking + +Command *etcdctl* + + [vagrant@localhost ~]$ etcdctl set /coreos.com/network/config '{ + > "Network": "10.120.0.0/15", + > "SubnetLen": 23, + > "SubnetMin": "10.120.0.0", + > "SubnetMax": "10.121.254.0", + > "Backend": { + > "Type": "udp" + > } + >}' + { + "Network": "10.120.0.0/15", + "SubnetLen": 23, + "SubnetMin": "10.120.0.0", + "SubnetMax": "10.121.254.0", + "Backend": { + "Type": "udp" + } + } + + [vagrant@localhost ~]$ etcdctl get /coreos.com/network + +* Set up 1st node + +NIC + + [vagrant@localhost ~]$ ip addr show eth1 + 3: eth1: mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether 08:00:27:c7:35:fe brd ff:ff:ff:ff:ff:ff + inet 10.64.33.81/24 brd 10.64.33.255 scope global eth1 + valid_lft forever preferred_lft forever + inet6 fe80::a00:27ff:fec7:35fe/64 scope link + valid_lft forever preferred_lft forever + +Opts + + [vagrant@localhost ~]$ sudo vi /etc/sysconfig/flanneld + # Flanneld configuration options + + # etcd url location. Point this to the server where etcd runs + # FLANNEL_ETCD="http://127.0.0.1:2379" + FLANNEL_ETCD="http://10.64.33.81:2379" + + # etcd config key. This is the configuration key that flannel queries + # For address range assignment + # FLANNEL_ETCD_KEY="/atomic.io/network" + FLANNEL_ETCD_KEY="/coreos.com/network" + + # Any additional options that you want to pass + #FLANNEL_OPTIONS="" + FLANNEL_OPTIONS="-iface=eth1" + +Service + + [vagrant@localhost ~]$ sudo systemctl edit --full flanneld.service + [Unit] + Description=Flanneld overlay address etcd agent + After=network.target + After=network-online.target + Wants=network-online.target + After=etcd.service + Before=docker.service + + [Service] + Type=notify + EnvironmentFile=/etc/sysconfig/flanneld + EnvironmentFile=-/etc/sysconfig/docker-network + ExecStart=/usr/bin/flanneld -etcd-endpoints=${FLANNEL_ETCD} -etcd-prefix=${FLANNEL_ETCD_KEY} $FLANNEL_OPTIONS + ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker + Restart=on-failure + + [Install] + WantedBy=multi-user.target + RequiredBy=docker.service + + [vagrant@localhost ~]$ sudo systemctl enable flanneld.service + Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /etc/systemd/system/flanneld.service. + Created symlink from /etc/systemd/system/docker.service.requires/flanneld.service to /etc/systemd/system/flanneld.service. + + [vagrant@localhost ~]$ sudo systemctl start flanneld.service + +Validation + + [vagrant@localhost ~]$ sudo systemctl -l status flanneld.service + 鈼▒ flanneld.service - Flanneld overlay address etcd agent + Loaded: loaded (/etc/systemd/system/flanneld.service; enabled; vendor preset: disabled) + Active: active (running) since Sat 2016-11-12 22:39:11 UTC; 7s ago + Process: 13347 ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker (code=exited, status=0/SUCCESS) + Main PID: 13341 (flanneld) + CGroup: /system.slice/flanneld.service + 鈹斺攢13341 /usr/bin/flanneld -etcd-endpoints=http://10.64.33.81:2379 -etcd-prefix=/coreos.com/network -iface=eth1 + + Nov 12 22:39:11 localhost.localdomain systemd[1]: Starting Flanneld overlay address etcd agent... + Nov 12 22:39:11 localhost.localdomain flanneld[13341]: I1112 22:39:11.698744 13341 main.go:275] Installing signal handlers + Nov 12 22:39:11 localhost.localdomain flanneld[13341]: I1112 22:39:11.699475 13341 main.go:188] Using 10.64.33.81 as external interface + Nov 12 22:39:11 localhost.localdomain flanneld[13341]: I1112 22:39:11.699490 13341 main.go:189] Using 10.64.33.81 as external endpoint + Nov 12 22:39:11 localhost.localdomain flanneld[13341]: I1112 22:39:11.701864 13341 etcd.go:139] Found lease (10.120.78.0/23) for current IP (10.64.33.81) but not compatible with current config, deleting + Nov 12 22:39:11 localhost.localdomain flanneld[13341]: I1112 22:39:11.702440 13341 etcd.go:204] Picking subnet in range 10.120.128.0 ... 10.123.128.0 + Nov 12 22:39:11 localhost.localdomain flanneld[13341]: I1112 22:39:11.703392 13341 etcd.go:84] Subnet lease acquired: 10.121.24.0/23 + Nov 12 22:39:11 localhost.localdomain flanneld[13341]: I1112 22:39:11.705016 13341 udp.go:222] Watching for new subnet leases + Nov 12 22:39:11 localhost.localdomain systemd[1]: Started Flanneld overlay address etcd agent. + + [vagrant@localhost ~]$ cat /run/flannel/subnet.env + FLANNEL_NETWORK=10.120.0.0/14 + FLANNEL_SUBNET=10.121.24.1/23 + FLANNEL_MTU=1472 + FLANNEL_IPMASQ=false + + [vagrant@localhost ~]$ cat /run/flannel/docker + DOCKER_OPT_BIP="--bip=10.121.24.1/23" + DOCKER_OPT_IPMASQ="--ip-masq=true" + DOCKER_OPT_MTU="--mtu=1472" + DOCKER_NETWORK_OPTIONS=" --bip=10.121.24.1/23 --ip-masq=true --mtu=1472 " + + [vagrant@localhost ~]$ cat /usr/lib/systemd/system/docker.service.d/flannel.conf + [Service] + EnvironmentFile=-/run/flannel/docker + + [vagrant@localhost ~]$ ip a + 1: lo: mtu 65536 qdisc noqueue state UNKNOWN + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: eth0: mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether 52:54:00:9f:bd:fd brd ff:ff:ff:ff:ff:ff + inet 10.0.2.15/24 brd 10.0.2.255 scope global dynamic eth0 + valid_lft 82840sec preferred_lft 82840sec + inet6 fe80::5054:ff:fe9f:bdfd/64 scope link + valid_lft forever preferred_lft forever + 3: eth1: mtu 1500 qdisc pfifo_fast state UP qlen 1000 + link/ether 08:00:27:c7:35:fe brd ff:ff:ff:ff:ff:ff + inet 10.64.33.81/24 brd 10.64.33.255 scope global eth1 + valid_lft forever preferred_lft forever + inet6 fe80::a00:27ff:fec7:35fe/64 scope link + valid_lft forever preferred_lft forever + 5: docker0: mtu 1500 qdisc noqueue state DOWN + link/ether 02:42:d0:2e:56:3f brd ff:ff:ff:ff:ff:ff + inet 10.121.24.1/23 scope global docker0 + valid_lft forever preferred_lft forever + 7: flannel0: mtu 1472 qdisc pfifo_fast state UNKNOWN qlen 500 + link/none + inet 10.121.24.0/14 scope global flannel0 + valid_lft forever preferred_lft forever + + [vagrant@localhost ~]$ ip r + default via 10.0.2.2 dev eth0 proto static metric 100 + 10.0.2.0/24 dev eth0 proto kernel scope link src 10.0.2.15 metric 100 + 10.64.33.0/24 dev eth1 proto kernel scope link src 10.64.33.81 + 10.120.0.0/14 dev flannel0 proto kernel scope link src 10.121.24.0 + 10.121.24.0/23 dev docker0 proto kernel scope link src 10.121.24.1 + 169.254.0.0/16 dev eth1 scope link metric 1003 + +* Command *nmcli* + + [vagrant@localhost ~]$ sudo nmcli d + DEVICE TYPE STATE CONNECTION + docker0 bridge connected docker0 + eth0 ethernet connected Wired connection 1 + flannel0 tun connected flannel0 + eth1 ethernet unmanaged -- + lo loopback unmanaged -- + + +* Set up 2nd node + +NIC + + [tangfx@localhost ~]$ ip a show enp0s8 + 3: enp0s8: mtu 1500 qdisc fq_codel state UP group default qlen 1000 + link/ether 08:00:27:24:4e:1c brd ff:ff:ff:ff:ff:ff + inet 10.64.33.90/24 brd 10.64.33.255 scope global enp0s8 + valid_lft forever preferred_lft forever + inet6 fe80::a00:27ff:fe24:4e1c/64 scope link + valid_lft forever preferred_lft forever + +Install into fedora23 + + [tangfx@localhost ~]$ sudo dnf install flannel + Last metadata expiration check: 0:00:40 ago on Sun Nov 13 07:16:02 2016. + Dependencies resolved. + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Installing: + flannel x86_64 0.5.5-3.fc23 updates 2.9 M + + Transaction Summary + ================================================================================ + Install 1 Package + +Opts + + [tangfx@localhost ~]$ sudo vi /etc/sysconfig/flanneld + # Flanneld configuration options + + # etcd url location. Point this to the server where etcd runs + # FLANNEL_ETCD="http://127.0.0.1:2379" + FLANNEL_ETCD="http://10.64.33.81:2379" + + # etcd config key. This is the configuration key that flannel queries + # For address range assignment + # FLANNEL_ETCD_KEY="/atomic.io/network" + FLANNEL_ETCD_KEY="/coreos.com/network" + + # Any additional options that you want to pass + #FLANNEL_OPTIONS="" + FLANNEL_OPTIONS="-iface=10.64.33.90" + +Service + + [tangfx@localhost ~]$ sudo systemctl enable flanneld.service + Created symlink from /etc/systemd/system/multi-user.target.wants/flanneld.service to /usr/lib/systemd/system/flanneld.service. + Created symlink from /etc/systemd/system/docker.service.requires/flanneld.service to /usr/lib/systemd/system/flanneld.service. + + [tangfx@localhost sysconfig]$ sudo systemctl start flanneld.service + +Validation + + [tangfx@localhost ~]$ cat /run/flannel/docker + DOCKER_OPT_BIP="--bip=10.120.204.1/23" + DOCKER_OPT_IPMASQ="--ip-masq=true" + DOCKER_OPT_MTU="--mtu=1472" + DOCKER_NETWORK_OPTIONS=" --bip=10.120.204.1/23 --ip-masq=true --mtu=1472 " + + [tangfx@localhost ~]$ cat /run/flannel/subnet.env + FLANNEL_NETWORK=10.120.0.0/14 + FLANNEL_SUBNET=10.120.204.1/23 + FLANNEL_MTU=1472 + FLANNEL_IPMASQ=false + + [tangfx@localhost ~]$ sudo systemctl -l status flanneld.service + 鈼▒ flanneld.service - Flanneld overlay address etcd agent + Loaded: loaded (/usr/lib/systemd/system/flanneld.service; enabled; vendor preset: disabled) + Active: active (running) since Sun 2016-11-13 07:21:23 CST; 21s ago + Process: 8573 ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker (code=exited, status=0/SUCCESS) + Main PID: 8563 (flanneld) + CGroup: /system.slice/flanneld.service + 鈹斺攢8563 /usr/bin/flanneld -etcd-endpoints=http://10.64.33.81:2379 -etcd-prefix=/coreos.com/network -iface=10.64.33.90 + + Nov 13 07:21:23 localhost.localdomain systemd[1]: Starting Flanneld overlay address etcd agent... + Nov 13 07:21:23 localhost.localdomain flanneld[8563]: I1113 07:21:23.661160 8563 main.go:275] Installing signal handlers + Nov 13 07:21:23 localhost.localdomain flanneld[8563]: I1113 07:21:23.661787 8563 main.go:188] Using 10.64.33.90 as external interface + Nov 13 07:21:23 localhost.localdomain flanneld[8563]: I1113 07:21:23.661843 8563 main.go:189] Using 10.64.33.90 as external endpoint + Nov 13 07:21:23 localhost.localdomain flanneld[8563]: I1113 07:21:23.667026 8563 etcd.go:204] Picking subnet in range 10.120.128.0 ... 10.123.128.0 + Nov 13 07:21:23 localhost.localdomain flanneld[8563]: I1113 07:21:23.668527 8563 etcd.go:84] Subnet lease acquired: 10.120.204.0/23 + Nov 13 07:21:23 localhost.localdomain flanneld[8563]: I1113 07:21:23.690345 8563 udp.go:222] Watching for new subnet leases + Nov 13 07:21:23 localhost.localdomain flanneld[8563]: I1113 07:21:23.692843 8563 udp.go:247] Subnet added: 10.121.24.0/23 + Nov 13 07:21:23 localhost.localdomain systemd[1]: Started Flanneld overlay address etcd agent. + + [tangfx@localhost ~]$ ip a show flannel0 + 10: flannel0: mtu 1472 qdisc fq_codel state UNKNOWN group default qlen 500 + link/none + inet 10.120.204.0/14 scope global flannel0 + valid_lft forever preferred_lft forever + inet6 fe80::edd6:165a:c078:f334/64 scope link flags 800 + valid_lft forever preferred_lft forever + + [tangfx@localhost ~]$ ip r + default via 10.0.2.2 dev enp0s3 proto static metric 100 + 10.0.2.0/24 dev enp0s3 proto kernel scope link src 10.0.2.15 metric 100 + 10.64.33.0/24 dev enp0s8 proto kernel scope link src 10.64.33.90 metric 100 + 10.120.0.0/23 dev br-34d5cc1180cb proto kernel scope link src 10.120.0.1 + 10.120.0.0/14 dev flannel0 proto kernel scope link src 10.120.204.0 + 172.17.0.0/16 dev lbr0 proto kernel scope link src 172.17.0.1 + +* Command *nmcli* + + [tangfx@localhost ~]$ sudo nmcli d + DEVICE TYPE STATE CONNECTION + br-34d5cc1180cb bridge connected br-34d5cc1180cb + lbr0 bridge connected lbr0 + enp0s3 ethernet connected enp0s3 + enp0s8 ethernet connected enp0s8 + flannel0 tun connected flannel0 + lo loopback unmanaged -- + br0 openvswitch unmanaged -- + ovs-system openvswitch unmanaged -- + tun0 openvswitch unmanaged -- + vxlan_sys_4789 vxlan unmanaged -- + + [tangfx@localhost ~]$ sudo nmcli con edit lbr0 + nmcli> set ipv4.addresses + Enter 'addresses' value: 10.120.204.1/23 + + nmcli> save persistent + Connection 'lbr0' (c0705f51-9954-4943-8361-af31ef6522db) successfully updated. + + nmcli> quit + + [tangfx@localhost ~]$ sudo nmcli con up lbr0 + Connection successfully activated (master waiting for slaves) (D-Bus active path: /org/freedesktop/NetworkManager/ActiveConnection/6) + + [tangfx@localhost ~]$ ip a show lbr0 + 4: lbr0: mtu 1500 qdisc noqueue state DOWN group default qlen 1000 + link/ether f2:41:f3:b1:a3:c3 brd ff:ff:ff:ff:ff:ff + inet 10.120.204.1/23 brd 10.120.205.255 scope global lbr0 + valid_lft forever preferred_lft forever + +* Cluster networking + +From 2nd PING 1st + + [tangfx@localhost ~]$ ping -c3 10.121.24.0 + PING 10.121.24.0 (10.121.24.0) 56(84) bytes of data. + 64 bytes from 10.121.24.0: icmp_seq=1 ttl=62 time=0.670 ms + 64 bytes from 10.121.24.0: icmp_seq=2 ttl=62 time=1.07 ms + 64 bytes from 10.121.24.0: icmp_seq=3 ttl=62 time=1.07 ms + + --- 10.121.24.0 ping statistics --- + 3 packets transmitted, 3 received, 0% packet loss, time 2001ms + rtt min/avg/max/mdev = 0.670/0.938/1.074/0.191 ms + + [tangfx@localhost ~]$ ping -c3 10.121.24.1 + PING 10.121.24.1 (10.121.24.1) 56(84) bytes of data. + 64 bytes from 10.121.24.1: icmp_seq=1 ttl=62 time=0.104 ms + 64 bytes from 10.121.24.1: icmp_seq=2 ttl=62 time=8.45 ms + 64 bytes from 10.121.24.1: icmp_seq=3 ttl=62 time=1.18 ms + + --- 10.121.24.1 ping statistics --- + 3 packets transmitted, 3 received, 0% packet loss, time 2001ms + rtt min/avg/max/mdev = 0.104/3.250/8.457/3.708 ms + +From 1st PING 2nd + + [vagrant@localhost ~]$ ping -c3 10.120.204.1 + PING 10.120.204.1 (10.120.204.1) 56(84) bytes of data. + 64 bytes from 10.120.204.1: icmp_seq=1 ttl=62 time=0.538 ms + 64 bytes from 10.120.204.1: icmp_seq=2 ttl=62 time=1.24 ms + 64 bytes from 10.120.204.1: icmp_seq=3 ttl=62 time=1.10 ms + + --- 10.120.204.1 ping statistics --- + 3 packets transmitted, 3 received, 0% packet loss, time 2002ms + rtt min/avg/max/mdev = 0.538/0.960/1.240/0.303 ms + + [vagrant@localhost ~]$ ping -c3 10.120.204.0 + PING 10.120.204.0 (10.120.204.0) 56(84) bytes of data. + 64 bytes from 10.120.204.0: icmp_seq=1 ttl=62 time=0.412 ms + 64 bytes from 10.120.204.0: icmp_seq=2 ttl=62 time=1.05 ms + 64 bytes from 10.120.204.0: icmp_seq=3 ttl=62 time=1.09 ms + + --- 10.120.204.0 ping statistics --- + 3 packets transmitted, 3 received, 0% packet loss, time 2001ms + rtt min/avg/max/mdev = 0.412/0.855/1.098/0.313 ms diff --git a/hack/kubernetes-clearwater/gofileserver/main.go b/hack/kubernetes-clearwater/gofileserver/main.go new file mode 100755 index 0000000..d766bb9 --- /dev/null +++ b/hack/kubernetes-clearwater/gofileserver/main.go @@ -0,0 +1,47 @@ +/* + Inspird by golang code func ExampleFileServer_stripPrefix() +*/ +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" + "runtime" +) + +var fileServer = struct { + addr string + dir string +}{} + +func main() { + flag.StringVar(&fileServer.addr, "addr", ":48080", "default listening 0.0.0.0:48080") + flag.StringVar(&fileServer.dir, "dir", "", "default is working directory") + flag.Parse() + + if len(os.Getenv("GOMAXPROCS")) == 0 { + runtime.GOMAXPROCS(runtime.NumCPU()) + } + + if len(fileServer.dir) == 0 { + var err error + fileServer.dir, err = os.Getwd() + if err != nil { + fmt.Printf("Unexpected of get working directory error: %+v\n", err) + os.Exit(1) + } + } + + http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(fileServer.dir)))) + + addr := ":48080" + if len(fileServer.addr) != 0 { + addr = fileServer.addr + } + + fmt.Println("Listening at ", addr) + log.Fatal(http.ListenAndServe(addr, nil)) +} diff --git a/hack/kubernetes-clearwater/kube/easy-rsa.tar.gz b/hack/kubernetes-clearwater/kube/easy-rsa.tar.gz new file mode 100644 index 0000000..0716878 Binary files /dev/null and b/hack/kubernetes-clearwater/kube/easy-rsa.tar.gz differ diff --git a/hack/kubernetes-clearwater/kubernetes-installation.md b/hack/kubernetes-clearwater/kubernetes-installation.md new file mode 100755 index 0000000..d96f29b --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes-installation.md @@ -0,0 +1,1622 @@ +Kubernetes DevOps +================== + +Tables of content +------------------ + +* Prerequisites + +* Binaries + +>* Download release + +>* Build from source + +* Generate CA and certs + +* Systemd services - master + +>* Install kube-apiserver + +>* Install kube-controller-manager server + +>* Install kube-scheduler server + +>* Install kubelet server + +>* Install kube-proxy server + +* POD image and container + +* Kubernetes addons + +>* Start kube-dns + +>* Start kubernetes-dashboard + +>* Issue + +* Install node - worker + +Prerequisites +-------------- + +Host firewall + + [vagrant@localhost ~]$ sudo systemctl disable firewalld.service + +Etcd must be installed + +Docker is required + +软件的可执行程序文件 +------------------- + +### Download release + +For example + + [vagrant@localhost ~]$ wget -c https://github.com/kubernetes/kubernetes/releases/download/v1.3.10/kubernetes.tar.gz + --2016-11-11 01:58:59-- https://github.com/kubernetes/kubernetes/releases/download/v1.3.10/kubernetes.tar.gz + Resolving github.com (github.com)... 192.30.253.113, 192.30.253.112 + Connecting to github.com (github.com)|192.30.253.113|:443... connected. + HTTP request sent, awaiting response... 302 Found + Location: https://github-cloud.s3.amazonaws.com/releases/20580498/b73a52cc-9f80-11e6-8085-e5bdc11f8bd8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAISTNZFOVBIJMK3TQ%2F20161111%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20161111T015900Z&X-Amz-Expires=300&X-Amz-Signature=77d9df5701f07bd4922654ac7aad99ba4a1aa65d32c2cc8c258170b710511645&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Dkubernetes.tar.gz&response-content-type=application%2Foctet-stream [following] + --2016-11-11 01:59:00-- https://github-cloud.s3.amazonaws.com/releases/20580498/b73a52cc-9f80-11e6-8085-e5bdc11f8bd8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAISTNZFOVBIJMK3TQ%2F20161111%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20161111T015900Z&X-Amz-Expires=300&X-Amz-Signature=77d9df5701f07bd4922654ac7aad99ba4a1aa65d32c2cc8c258170b710511645&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Dkubernetes.tar.gz&response-content-type=application%2Foctet-stream + Resolving github-cloud.s3.amazonaws.com (github-cloud.s3.amazonaws.com)... 52.216.0.128 + Connecting to github-cloud.s3.amazonaws.com (github-cloud.s3.amazonaws.com)|52.216.0.128|:443... connected. + HTTP request sent, awaiting response... 206 Partial Content + Length: 1495467451 (1.4G), 1495466858 (1.4G) remaining [application/octet-stream] + Saving to: 鈥榢ubernetes.tar.gz鈥▒ + + 27% [=========> ] 405,582,941 3.21MB/s eta 8m 34s + + [vagrant@localhost ~]$ sudo tar -C /opt -zxf kubernetes.tar.gz kubernetes/server/kubernetes-server-linux-amd64.tar.gz + + [vagrant@localhost ~]$ sudo tar -tzf /opt/kubernetes/server/kubernetes-server-linux-amd64.tar.gz + kubernetes/ + kubernetes/addons/ + kubernetes/kubernetes-src.tar.gz + kubernetes/server/ + kubernetes/server/bin/ + kubernetes/server/bin/kube-proxy.tar + kubernetes/server/bin/kube-controller-manager + kubernetes/server/bin/kube-proxy.docker_tag + kubernetes/server/bin/federation-apiserver.docker_tag + kubernetes/server/bin/federation-controller-manager.docker_tag + kubernetes/server/bin/kube-scheduler + kubernetes/server/bin/kube-scheduler.docker_tag + kubernetes/server/bin/kube-controller-manager.tar + kubernetes/server/bin/kubectl + kubernetes/server/bin/kube-apiserver.docker_tag + kubernetes/server/bin/federation-apiserver.tar + kubernetes/server/bin/kube-apiserver.tar + kubernetes/server/bin/federation-controller-manager.tar + kubernetes/server/bin/federation-apiserver + kubernetes/server/bin/kube-dns + kubernetes/server/bin/kubemark + kubernetes/server/bin/kubelet + kubernetes/server/bin/kube-proxy + kubernetes/server/bin/kube-scheduler.tar + kubernetes/server/bin/kube-apiserver + kubernetes/server/bin/kube-controller-manager.docker_tag + kubernetes/server/bin/federation-controller-manager + kubernetes/server/bin/hyperkube + kubernetes/LICENSES + + [vagrant@localhost ~]$ sudo tar -C /opt -zxf /opt/kubernetes/server/kubernetes-server-linux-amd64.tar.gz + + [vagrant@localhost ~]$ sudo ls /opt/kubernetes/ + 1.3.10 addons kubernetes-src.tar.gz LICENSES server + +### Build (development option) + +使用git + + [tangfx@localhost k8s.io]$ git clone https://github.com/tangfeixiong/kubernetes kubernetes + Cloning into 'kubernetes'... + remote: Counting objects: 337701, done. + remote: Compressing objects: 100% (5/5), done. + Receiving objects: 19% (65526/337701), 72.23 MiB | 127.00 KiB/s + + [tangfx@localhost k8s.io]$ git remote add upstream https://github.com/kubernetes/kubernetes + + [tangfx@localhost k8s.io]$ git pull upstream + + [tangfx@localhost kubernetes]$ git tag --list + +检出v1.3.10 + + [tangfx@localhost kubernetes]$ git checkout v1.3.10 + Note: checking out 'v1.3.10'. + + You are in 'detached HEAD' state. You can look around, make experimental + changes and commit them, and you can discard any commits you make in this + state without impacting any branches by performing another checkout. + + If you want to create a new branch to retain commits you create, you may + do so (now or later) by using -b with the checkout command again. Example: + + git checkout -b + + HEAD is now at c3e367e... Kubernetes version v1.3.10 + + [tangfx@localhost kubernetes]$ git status + HEAD detached at v1.3.10 + nothing to commit, working directory clean + +Golang语言 + + [tangfx@localhost kubernetes]$ go version + go version go1.6.2 linux/amd64 + +Make + + [tangfx@localhost kubernetes]$ make all GOFLAGS=-v + hack/build-go.sh + Go version: go version go1.6.2 linux/amd64 + +++ [1109 15:43:03] Building the toolchain targets: + k8s.io/kubernetes/hack/cmd/teststale + k8s.io/kubernetes/vendor/github.com/golang/glog + k8s.io/kubernetes/hack/cmd/teststale + +++ [1109 15:43:05] Building go targets for linux/amd64: + cmd/kube-dns + cmd/kube-proxy + cmd/kube-apiserver + cmd/kube-controller-manager + cmd/kubelet + cmd/kubemark + cmd/hyperkube + federation/cmd/federation-apiserver + federation/cmd/federation-controller-manager + plugin/cmd/kube-scheduler + cmd/kubectl + cmd/integration + cmd/gendocs + cmd/genkubedocs + cmd/genman + cmd/genyaml + cmd/mungedocs + cmd/genswaggertypedocs + cmd/linkcheck + examples/k8petstore/web-server/src + federation/cmd/genfeddocs + vendor/github.com/onsi/ginkgo/ginkgo + test/e2e/e2e.test + test/e2e_node/e2e_node.test + +++ [1109 15:43:05] +++ Warning: stdlib pkg with cgo flag not found. + +++ [1109 15:43:05] +++ Warning: stdlib pkg cannot be rebuilt since /usr/local/go/pkg is not writable by tangfx + +++ [1109 15:43:05] +++ Warning: Make /usr/local/go/pkg writable for tangfx for a one-time stdlib install, Or + +++ [1109 15:43:05] +++ Warning: Rebuild stdlib using the command 'CGO_ENABLED=0 go install -a -installsuffix cgo std' + +++ [1109 15:43:05] +++ Falling back to go build, which is slower + ********************** + +++ [1109 15:41:00] Placing binaries + + [vagrant@localhost kubernetes]$ ls _output/bin/ + conversion-gen integration + deepcopy-gen kube-apiserver + defaulter-gen kube-controller-manager + e2e_node.test kubectl + e2e.test kube-dns + federation-apiserver kubelet + federation-controller-manager kubemark + gendocs kube-proxy + genfeddocs kube-scheduler + genkubedocs linkcheck + genman mungedocs + genswaggertypedocs openapi-gen + genyaml src + ginkgo teststale + hyperkube + + [vagrant@localhost kubernetes]$ _output/bin/kubelet --version + Kubernetes v1.3.10-dirty + + [vagrant@localhost kubernetes]$ _output/bin/kubectl version --client + Client Version: version.Info{Major:"1", Minor:"3+", GitVersion:"v1.3.10-dirty", GitCommit:"c3e367ec9eae7338ac4e2a57f293634891319b7c", GitTreeState:"dirty", BuildDate:"2016-11-09T15:11:23Z", GoVersion:"go1.6.2", Compiler:"gc", Platform:"linux/amd64"} + +Place binaries + + [vagrant@localhost ~]$ sudo chown vagrant: /opt/kubernetes/1.3.10 + + [vagrant@localhost ~]$ mkdir bin + + [vagrant@localhost ~]$ ln -s /opt/kubernetes/1.3.10/{kubectl,kubelet,kube-apiserver,kube-controller-manager,kube-scheduler,kube-proxy,hyperkube} bin + +Generate CA and Certs +------------------------ + +Using [saltbase make-ca-cert.sh](https://github.com/kubernetes/kubernetes/tree/master/cluster/saltbase/salt/generate-cert) + +easy-rsa reference => https://github.com/OpenVPN/easy-rsa + + [vagrant@localhost ~]$ ls . kube + .: + bin kube kubernetes salt-make-ca-cert.sh + + kube: + easy-rsa.tar.gz + + [vagrant@localhost ~]$ ./salt-make-ca-cert.sh + IP:10.64.33.81,IP:10.123.240.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local + Create ca certs into /srv/kubernetes + + [vagrant@localhost ~]$ ls /srv/kubernetes + ca.crt kubecfg.crt kubecfg.key server.cert server.key + +Systemd service +---------------- + +### Run all components as systemd services + +Reference + + https://github.com/kubernetes/kubernetes/tree/master/cluster/centos/master/scripts + +* The kube-apiserver + +Service file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/kube-apiserver.service + [Unit] + Description=Kubernetes API server + Documentation=https://github.com/kubernetes/kubernetes + After=etcd.service + Requires=etcd.service + # Wants= + # Conflicts=openshift-master.service + + [Service] + Type=notify + NotifyAccess=all + # Type=simple + User=root + + Environment=KUBERNETES_BUILD_VERSION=1.3.10 + EnvironmentFile=/etc/sysconfig/kube-apiserver + WorkingDirectory=/opt/kubernetes + + ExecStart=/opt/kubernetes/server/bin/kube-apiserver $KUBE_APISERVER_OPTS + # ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-apiserver ${KUBE_APISERVER_OPTS}" + + Restart=always + # Restart=on-failure + + [Install] + WantedBy=multi-user.target + +Opts file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/conf/kube-apiserver + # KUBE_APISERVER_VERSION=1.3.10 + + # --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota + + # --advertise-address=: IP address to advertise the apiserver to members of the cluster, must be reachable by the cluster + # ADVERTISE_APISERVER=10.64.33.81 + + # --cert-dir="/var/run/kubernetes" + # CERT_DIR=/srv/kubernetes + + # --enable-swagger-ui[=false]: Enables swagger ui at /swagger-ui + # ENABLE_SWAGGER_UI=false + + # --etcd-servers=: comma separated etcd servers to connect (http://ip:port) + # ETCD_SERVERS=http://10.64.33.81:2379 + + # --master-service-namespace="default": in which the kubernetes master services should be injected into pods + + # --runtime-config=: A set of key=value pairs such as apis/=true/false, also apis// + # RUNTIME_CONFIG=api/all=true + + # --secure-port=6443: The port on which to serve HTTPS + # SECURE_PORT=6443 + + # --service-cluster-ip-range=: A CIDR notation IP range + # Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 + # CLUSTER_CIDR=10.120.0.0/14 + # SERVICE_CIDR=10.123.240.0/20 + # CLUSTER_MASTER=10.123.240.1 + # CLUSTER_DNS=10.123.240.10 + + # --service-node-port-range=: default '30000-32767' + + KUBE_APISERVER_OPTS="--admission-control=AlwaysAdmit \ + --advertise-address=10.64.33.81 \ + --allow-privileged=true \ + --apiserver-count=1 \ + --bind-address=0.0.0.0 \ + --cert-dir=/srv/kubernetes \ + --client-ca-file=/srv/kubernetes/ca.crt \ + --enable-swagger-ui=false \ + --etcd-servers=http://10.64.33.81:2379 \ + --insecure-bind-address=127.0.0.1 \ + --insecure-port=8080 \ + --master-service-namespace=default \ + --runtime-config=api/all=true \ + --secure-port=6443 \ + --service-cluster-ip-range=10.123.240.0/20 \ + --tls-cert-file=/srv/kubernetes/server.crt \ + --tls-private-key-file=/srv/kubernetes/server.key \ + --v=2" + +Manually install + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/conf/kube-apiserver /etc/sysconfig/ + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/system/kube-apiserver.service /etc/systemd/system + + [vagrant@localhost ~]$ sudo systemctl enable kube-apiserver.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /etc/systemd/system/kube-apiserver.service. + +Experimental run + + [vagrant@localhost ~]$ KUBE_APISERVER_OPTS="--admission-control=AlwaysAdmit \ + > --advertise-address=10.64.33.81 \ + > --allow-privileged=true \ + > --apiserver-count=1 \ + > --bind-address=0.0.0.0 \ + > --cert-dir=/srv/kubernetes \ + > --client-ca-file=/srv/kubernetes/ca.crt \ + > --enable-swagger-ui=false \ + > --etcd-servers=http://10.64.33.81:2379 \ + > --insecure-bind-address=127.0.0.1 \ + > --insecure-port=8080 \ + > --master-service-namespace=default \ + > --runtime-config=api/all=true \ + > --secure-port=6443 \ + > --service-cluster-ip-range=10.123.240.0/20 \ + > --tls-cert-file=/srv/kubernetes/server.crt \ + > --tls-private-key-file=/srv/kubernetes/server.key \ + > --v=2"; sudo /opt/kubernetes/server/bin/kube-apiserver $KUBE_APISERVER_OPTS + +First start + + [vagrant@localhost ~]$ sudo systemctl daemon-reload + + [vagrant@localhost ~]$ sudo systemctl start kube-apiserver.service + +Validation + + [vagrant@localhost ~]$ sudo systemctl -l status kube-apiserver.service + 鈼▒ kube-apiserver.service - Kubernetes API server + Loaded: loaded (/etc/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled) + Active: active (running) since Thu 2016-11-10 22:43:54 UTC; 19s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 5842 (kube-apiserver) + CGroup: /system.slice/kube-apiserver.service + 鈹斺攢5842 /opt/kubernetes/1.3.10/kube-apiserver --admission-control=AlwaysAdmit --advertise-address=10.64.33.81 --allow-privileged=true --apiserver-count=1 --bind-address=0.0.0.0 --cert-dir=/srv/kubernetes --client-ca-file=/srv/kubernetes/ca.crt --enable-swagger-ui=false --etcd-servers=http://10.64.33.81:2379 --insecure-bind-address=127.0.0.1 --insecure-port=8080 --master-service-namespace=default --runtime-config=api/all=true --secure-port=6443 --service-cluster-ip-range=10.123.240.0/20 --tls-cert-file=/srv/kubernetes/server.cert --tls-private-key-file=/srv/kubernetes/server.key --v=2 + + Nov 10 22:43:54 localhost.localdomain systemd[1]: Starting Kubernetes API server... + Nov 10 22:43:54 localhost.localdomain kube-apiserver[5842]: I1110 22:43:54.411559 5842 genericapiserver.go:606] Will report 10.64.33.81 as public IP address. + Nov 10 22:43:54 localhost.localdomain kube-apiserver[5842]: I1110 22:43:54.412884 5842 genericapiserver.go:288] Node port range unspecified. Defaulting to 30000-32767. + Nov 10 22:43:54 localhost.localdomain kube-apiserver[5842]: [restful] 2016/11/10 22:43:54 log.go:30: [restful/swagger] listing is available at https://10.64.33.81:6443/swaggerapi/ + Nov 10 22:43:54 localhost.localdomain kube-apiserver[5842]: [restful] 2016/11/10 22:43:54 log.go:30: [restful/swagger] https://10.64.33.81:6443/swaggerui/ is mapped to folder /swagger-ui/ + Nov 10 22:43:54 localhost.localdomain kube-apiserver[5842]: I1110 22:43:54.505181 5842 genericapiserver.go:690] Serving securely on 0.0.0.0:6443 + Nov 10 22:43:54 localhost.localdomain kube-apiserver[5842]: I1110 22:43:54.505191 5842 genericapiserver.go:734] Serving insecurely on 127.0.0.1:8080 + Nov 10 22:43:54 localhost.localdomain systemd[1]: Started Kubernetes API server. + + [vagrant@localhost ~]$ sudo journalctl --follow --no-pager --pager-end --no-tail -u kube-apiserver.service + + [vagrant@localhost ~]$ sudo tail -100 /var/log/messages + +Deep dive into kube-apiserver + + [vagrant@localhost ~]$ etcdctl ls + /registry + [vagrant@localhost ~]$ etcdctl ls /registry + /registry/namespaces + /registry/services + /registry/ranges + [vagrant@localhost ~]$ etcdctl ls /registry/ranges + /registry/ranges/servicenodeports + /registry/ranges/serviceips + [vagrant@localhost ~]$ etcdctl get /registry/ranges/serviceips + {"kind":"RangeAllocation","apiVersion":"v1","metadata":{"creationTimestamp":null},"range":"10.123.240.0/20","data":"AQ=="} + + [vagrant@localhost ~]$ curl http://127.0.0.1:8080/ + { + "paths": [ + "/api", + "/api/v1", + "/apis", + "/apis/apps", + "/apis/apps/v1alpha1", + "/apis/autoscaling", + "/apis/autoscaling/v1", + "/apis/batch", + "/apis/batch/v1", + "/apis/batch/v2alpha1", + "/apis/extensions", + "/apis/extensions/v1beta1", + "/apis/policy", + "/apis/policy/v1alpha1", + "/apis/rbac.authorization.k8s.io", + "/apis/rbac.authorization.k8s.io/v1alpha1", + "/healthz", + "/healthz/ping", + "/logs/", + "/metrics", + "/swaggerapi/", + "/ui/", + "/version" + ] + + [vagrant@localhost ~]$ sudo curl --cacert /srv/kubernetes/ca.crt --cert /srv/kuernetes/kubecfg.crt --key /srv/kubernetes/kubecfg.key https://10.64.33.81:6443/api + { + "kind": "APIVersions", + "versions": [ + "v1" + ], + "serverAddressByClientCIDRs": [ + { + "clientCIDR": "0.0.0.0/0", + "serverAddress": "10.64.33.81:6443" + } + ] + } + +Command *kubectl* + + [vagrant@localhost ~]$ kubectl config set-cluster kube --server=https://10.64.33.81:6443 + cluster "kube" set. + + [vagrant@localhost ~]$ encoded=$(sudo base64 -w0 /srv/kubernetes/ca.crt); kubectl config set clusters.kube.certificate-authority-data "$encoded" + property "clusters.kube.certificate-authority-data" set. + + [vagrant@localhost ~]$ kubectl config set-credentials admin + user "admin" set. + + [vagrant@localhost ~]$ encoded=$(sudo base64 -w0 /srv/kubernetes/kubecfg.crt); kubectl config set users.admin.client-certificate-data "$encoded" + property "users.admin.client-certificate-data" set. + + [vagrant@localhost ~]$ encoded=$(sudo base64 -w0 /srv/kubernetes/kubecfg.key); kubectl config set users.admin.client-key-data "$encoded" + property "users.admin.client-key-data" set. + + [vagrant@localhost ~]$ kubectl config set-context kube-admin --cluster=kube --user=admin + context "kube-admin" set. + + [vagrant@localhost ~]$ kubectl config use-context kube-admin + switched to context "kube-admin". + + [vagrant@localhost ~]$ cat .kube/config + + [vagrant@localhost ~]$ kubectl version + Client Version: version.Info{Major:"1", Minor:"3+", GitVersion:"v1.3.10-dirty", GitCommit:"c3e367ec9eae7338ac4e2a57f293634891319b7c", GitTreeState:"dirty", BuildDate:"2016-11-09T15:11:23Z", GoVersion:"go1.6.2", Compiler:"gc", Platform:"linux/amd64"} + Server Version: version.Info{Major:"1", Minor:"3+", GitVersion:"v1.3.10-dirty", GitCommit:"c3e367ec9eae7338ac4e2a57f293634891319b7c", GitTreeState:"dirty", BuildDate:"2016-11-09T15:11:23Z", GoVersion:"go1.6.2", Compiler:"gc", Platform:"linux/amd64"} + + [vagrant@localhost ~]$ kubectl api-versions + apps/v1alpha1 + autoscaling/v1 + batch/v1 + batch/v2alpha1 + extensions/v1beta1 + policy/v1alpha1 + rbac.authorization.k8s.io/v1alpha1 + v1 + +* The kube-controller-manager + +Service file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/system/kube-controller-manager.service + [Unit] + Description=Kubernetes controller manager + Documentation=https://github.com/kubernetes/kubernetes + # After= + # Requires= + # Wants= + # Conflicts=openshift-master.service + + [Service] + # Type=notify + # NotifyAccess=none + Type=simple + User=root + + Environment=KUBERNETES_BUILD_VERSION=1.3.10 + EnvironmentFile=/etc/sysconfig/kube-controller-manager + WorkingDirectory=/opt/kubernetes + + ExecStart=/opt/kubernetes/server/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS + # ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-controller-manager ${KUBE_CONTROLLER_MANAGER_OPTS}" + + # Restart=always + Restart=on-failure + + [Install] + WantedBy=multi-user.target + +Opts file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/conf/kube-controller-manager + # KUBE_CONTROLLER_MANAGER_VERSION=1.3.10 + + # ADDRESS_LISTEN=10.64.33.81 + + # --cloud-config="": The path to the cloud provider configuration file + + # --cloud-provider="": The provider for cloud services + + # MASTER_ADDRESS=https://10.64.33.81:6443 + + # --service-cluster-ip-range=: A CIDR notation IP range + # Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 + # CLUSTER_CIDR=10.120.0.0/14 + # SERVICE_CIDR=10.123.240.0/20 + # CLUSTER_MASTER=10.123.240.1 + # CLUSTER_DNS=10.123.240.10 + + KUBE_CONTROLLER_MANAGER_OPTS="--address=10.64.33.81 \ + --cluster-cidr=10.120.0.0/14 \ + --cluster-name=kubernetes \ + --enable-garbage-collector=false \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --port=10252 \ + --root-ca-file=/srv/kubernetes/ca.crt \ + --service-account-private-key-file=/srv/kubernetes/server.key \ + --service-cluster-ip-range=10.123.240.0/20 \ + --v=2" + +Manually install + + [vagrant@localhost ~]$ sudo cp .kube/config /srv/kubernetes/kubeconfig + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/conf/kube-controller-manager /etc/sysconfig/ + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/system/kube-controller-manager.service /etc/systemd/system + + [vagrant@localhost ~]$ sudo systemctl enable kube-controller-manager.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /etc/systemd/system/kube-controller-manager.service. + +Experimental run + + [vagrant@localhost ~]$ KUBE_CTLMGR_OPTS="--address=0.0.0.0 \ + > --cluster-cidr=10.120.0.0/14 \ + > --cluster-name=kubernetes \ + > --enable-garbage-collector=false \ + > --kubeconfig=/srv/kubernetes/kubeconfig \ + > --master=https://10.64.33.81:6443 \ + > --port=10252 \ + > --root-ca-file=/srv/kubernetes/ca.crt \ + > --service-account-private-key-file=/srv/kubernetes/server.key \ + > --service-cluster-ip-range=10.123.240.0/20 \ + > --v=2"; sudo /opt/kubernetes/server/bin/kube-controller-manager $KUBE_CTLMGR_OPTS + +First start + + [vagrant@localhost ~]$ sudo systemctl daemon-reload + + [vagrant@localhost ~]$ sudo systemctl start kube-controller-manager.service + +Validation + + [vagrant@localhost ~]$ sudo systemctl -l status kube-controller-manager.service + 鈼▒ kube-controller-manager.service - Kubernetes controller manager + Loaded: loaded (/etc/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled) + Active: active (running) since Sat 2016-11-12 16:15:45 UTC; 9s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 22499 (kube-controller) + CGroup: /system.slice/kube-controller-manager.service + 鈹斺攢22499 /opt/kubernetes/server/bin/kube-controller-manager --address=10.64.33.81 --cluster-cidr=10.120.0.0/14 --cluster-name=kubernetes --enable-garbage-collector=false --kubeconfig=/srv/kubernetes/kubeconfig --master=https://10.64.33.81:6443 --port=10252 --root-ca-file=/srv/kubernetes/ca.crt --service-account-private-key-file=/srv/kubernetes/server.key --service-cluster-ip-range=10.123.240.0/20 --v=2 + + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: I1112 16:15:45.722320 22499 plugins.go:340] Loaded volume plugin "kubernetes.io/aws-ebs" + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: I1112 16:15:45.722334 22499 plugins.go:340] Loaded volume plugin "kubernetes.io/gce-pd" + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: I1112 16:15:45.722340 22499 plugins.go:340] Loaded volume plugin "kubernetes.io/cinder" + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: I1112 16:15:45.722346 22499 plugins.go:340] Loaded volume plugin "kubernetes.io/vsphere-volume" + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: E1112 16:15:45.723734 22499 util.go:45] Metric for serviceaccount_controller already registered + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: I1112 16:15:45.724525 22499 attach_detach_controller.go:191] Starting Attach Detach Controller + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: W1112 16:15:45.725101 22499 request.go:347] Field selector: v1 - serviceaccounts - metadata.name - default: need to check if this is versioned correctly. + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: I1112 16:15:45.730257 22499 endpoints_controller.go:322] Waiting for pods controller to sync, requeuing service default/kubernetes + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: W1112 16:15:45.799771 22499 request.go:347] Field selector: v1 - serviceaccounts - metadata.name - default: need to check if this is versioned correctly. + Nov 12 16:15:45 localhost.localdomain kube-controller-manager[22499]: I1112 16:15:45.848602 22499 endpoints_controller.go:322] Waiting for pods controller to sync, requeuing service default/kubernetes + [vagrant@localhost ~]$ + + [vagrant@localhost ~]$ sudo journalctl --no-pager --no-tail -u kube-controller-manager.service + + [vagrant@localhost ~]$ sudo tail -100 /var/log/messages + +Investigate logs + + E1112 03:26:22.687235 11466 controllermanager.go:253] Failed to start service controller: ServiceController should not be run without a cloudprovider. + + E1112 03:26:22.690562 11466 util.go:45] Metric for replenishment_controller already registered + +* The kube-scheduler + +Service file + + [vagrant@localhost ~]$ vi kubernetes/centos/systemd/system/kube-scheduler.service + [Unit] + Description=Kubernetes scheduler + Documentation=https://github.com/kubernetes/kubernetes + # After= + # Requires= + # Wants= + # Conflicts=openshift-master.service + + [Service] + # Type=notify + # NotifyAccess=main + Type=simple + User=root + + Environment=KUBE_BUILD_VERSION=1.3.10 + EnvironmentFile=/etc/sysconfig/kube-scheduler + WorkingDirectory=/opt/kubernetes + + ExecStart=/opt/kubernetes/server/bin/kube-scheduler $KUBE_SCHEDULER_OPTS + # ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-scheduler ${KUBE_SCHEDULER_OPTS}" + + # Restart=always + Restart=on-failure + + [Install] + WantedBy=multi-user.target + +Opts file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/conf/kube-scheduler + # KUBE_SCHEDULER_VERSION=1.3.10 + + # ADDRESS_LISTEN=10.64.33.81 + # MASTER_ADDRESS=https://10.64.33.81:6443 + + # --cloud-config="": The path to the cloud provider configuration file + # --cloud-provider="": The provider for cloud services + + # --service-cluster-ip-range=: A CIDR notation IP range + # Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 + # CLUSTER_CIDR=10.120.0.0/14 + # SERVICE_CIDR=10.123.240.0/20 + # CLUSTER_MASTER=10.123.240.1 + # CLUSTER_DNS=10.123.240.10 + + KUBE_SCHEDULER_OPTS="--address=10.64.33.81 \ + --algorithm-provider=DefaultProvider \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --port=10251 \ + --scheduler-name=default-scheduler \ + --v=2" + +Manually install + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/conf/kube-scheduler /etc/systemd/ + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/system/kube-scheduler.service /etc/systemd/system + + [vagrant@localhost ~]$ sudo systemctl enable kube-scheduler + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /etc/systemd/system/kube-scheduler.service. + +Experimental run + + [vagrant@localhost ~]$ KUBE_SCHEDULER_OPTS="--address=0.0.0.0 \ + > --algorithm-provider=DefaultProvider \ + > --kubeconfig=/srv/kubernetes/kubeconfig \ + > --master=https://10.64.33.81:6443 \ + > --port=10251 \ + > --scheduler-name=default-scheduler \ + > --v=2"; sudo /opt/kubernetes/server/bin/kube-scheduler $KUBE_SCHEDULER_OPTS + +First start + + [vagrant@localhost ~]$ sudo systemctl daemon-reload + + [vagrant@localhost ~]$ sudo systemctl start kube-scheduler.service + +Validation + + [vagrant@localhost ~]$ sudo systemctl -l status kube-scheduler.service + 鈼▒ kube-scheduler.service - Kubernetes scheduler + Loaded: loaded (/etc/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled) + Active: active (running) since Sat 2016-11-12 16:34:52 UTC; 9s ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 23697 (kube-scheduler) + CGroup: /system.slice/kube-scheduler.service + 鈹斺攢23697 /opt/kubernetes/server/bin/kube-scheduler --address=10.64.33.81 --algorithm-provider=DefaultProvider --kubeconfig=/srv/kubernetes/kubeconfig --master=https://10.64.33.81:6443 --port=10251 --scheduler-name=default-scheduler --v=2 + + Nov 12 16:34:52 localhost.localdomain systemd[1]: Started Kubernetes scheduler. + Nov 12 16:34:52 localhost.localdomain systemd[1]: Starting Kubernetes scheduler... + Nov 12 16:34:52 localhost.localdomain kube-scheduler[23697]: I1112 16:34:52.405887 23697 factory.go:255] Creating scheduler from algorithm provider 'DefaultProvider' + Nov 12 16:34:52 localhost.localdomain kube-scheduler[23697]: I1112 16:34:52.405941 23697 factory.go:301] creating scheduler with fit predicates 'map[NoVolumeZoneConflict:{} MaxEBSVolumeCount:{} MaxGCEPDVolumeCount:{} GeneralPredicates:{} PodToleratesNodeTaints:{} CheckNodeMemoryPressure:{} NoDiskConflict:{}]' and priority functions 'map[SelectorSpreadPriority:{} NodeAffinityPriority:{} TaintTolerationPriority:{} LeastRequestedPriority:{} BalancedResourceAllocation:{}] + + [vagrant@localhost ~]$ sudo journalctl --follow --no-pager --pager-end --no-tail -u kube-scheduler.service + + [vagrant@localhost ~]$ sudo cat /var/log/messages | grep kube-scheduler + +* The kubelet + +User + + [vagrant@localhost ~]$ sudo adduser --home-dir=/var/lib/kubelet --system kubelet + + [vagrant@localhost ~]$ sudo usermod --shell /sbin/nologin kubelet + + [vagrant@localhost ~]$ sudo usermod -aG dockerroot kubelet + + [vagrant@localhost ~]$ sudo ls -Zd /var/lib/kubelet + drwxr-x---. root root system_u:object_r:var_lib_t:s0 /var/lib/kubelet + + [vagrant@localhost ~]$ sudo chcon -v --type=docker_var_lib_t /var/lib/kubelet + changing security context of 鈥▒/var/lib/kubelet鈥▒ + + [vagrant@localhost ~]$ sudo chcon -v --type=docker_var_lib_t /var/lib/kubelet/pods + changing security context of 鈥▒/var/lib/kubelet/pods鈥▒ + + [vagrant@localhost ~]$ sudo chcon -v --type=docker_var_lib_t /var/lib/kubelet/plugins + changing security context of 鈥▒/var/lib/kubelet/plugins鈥▒ + + [vagrant@localhost ~]$ sudo restorecon -Rv -n /var/lib/kubelet + restorecon reset /var/lib/kubelet context system_u:object_r:var_lib_t:s0->system_u:object_r:docker_var_lib_t:s0 + + [vagrant@localhost ~]$ sudo ls -Zd /var/lib/kubelet + drwxr-x---. root root system_u:object_r:docker_var_lib_t:s0 /var/lib/kubelet + + [vagrant@localhost addons]$ sudo semanage port -l + + [vagrant@localhost ~]$ sudo chgrp -R dockerroot /var/lib/kubelet + + [vagrant@localhost ~]$ sudo usermod -aG kube-cert dockerroot + + [vagrant@localhost ~]$ sudo chgrp dockerroot /srv/kubernetes + + [vagrant@localhost ~]$ sudo vi /etc/sysconfig/network-scripts/ifcfg-eth1 + # The contents below are automatically generated by Vagrant. Do not modify. + NM_CONTROLLED=no + BOOTPROTO=none + ONBOOT=yes + IPADDR=10.64.33.81 + NETMASK=255.255.255.0 + DEVICE=eth1 + PEERDNS=no + + [vagrant@localhost ~]$ sudo nmcli c edit eth1 + HWADDR=08:00:27:C7:35:FE + TYPE=Ethernet + BOOTPROTO=none + DEFROUTE=yes + IPV4_FAILURE_FATAL=no + IPV6INIT=yes + IPV6_AUTOCONF=yes + IPV6_DEFROUTE=yes + IPV6_FAILURE_FATAL=no + NAME=eth1 + UUID=de826ad3-ace9-4777-8b6f-3f935357fa85 + DEVICE=eth1 + ONBOOT=yes + IPADDR=10.64.33.81 + PREFIX=24 + DNS1=8.8.4.4 + DNS2=114.114.114.114 + IPV6_PEERDNS=yes + IPV6_PEERROUTES=yes + + [vagrant@localhost ~]$ sudo vi /etc/NetworkManager/NetworkManager.conf + [main] + plugins=ifcfg-rh + dns=dnsmasq + + [vagrant@localhost ~]$ sudo hostnamectl set-hostname master81 + + [vagrant@localhost ~]$ hostname + master81 + + [vagrant@localhost ~]$ cat /etc/hostname + master81 + + + + + + +Service file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/system/kubelet.service + [Unit] + Description=Kubernetes container mgmt daemon - kubelet + Documentation=https://github.com/kubernetes/kubernetes + After=docker.service + Requires=docker.service + # Wants= + Conflicts=openshift-node.service + + [Service] + # Type=notify + # NotifyAccess=all + Type=simple + User=root + + Environment=KUBERNETES_BUILD_VERSION=1.3.10 + EnvironmentFile=/etc/sysconfig/kubelet + WorkingDirectory=/opt/kubernetes + + ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/usr/bin/mkdir -p /var/log/containers + + ExecStart=/opt/kubernetes/server/bin/kubelet $KUBELET_OPTS + # ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kubelet ${KUBELET_OPTS}" + + # Restart=always + Restart=on-failure + RestartSec=10 + KillMode=process + + [Install] + WantedBy=multi-user.target + +Opts file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/conf/kubelet + # KUBELET_VERSION=1.3.10 + + # --api-servers=[]: Comma separated list of Kubernetes API servers (ip:port). + # API_SERVERS=https://10.64.33.81:6443 + + # --hostname-override="": As identification instad of the actual hostname + # HOSTNAME_OVERRIDE=10.64.33.81 + + # --cert-dir="/var/run/kubernetes" + # CERT_DIR=/srv/kubernetes + + # CONFIG_PATH=/etc/kubernetes/manifests + + # --kubeconfig="/var/lib/kubelet/kubeconfig": Path to a kubeconfig file + # KUBECONFIG_PATH=/srv/kubernetes/kubeconfig + + # --master-service-namespace="default": The master services should be injected into pods + + # --service-cluster-ip-range=: A CIDR notation IP range + # Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 + # CLUSTER_CIDR=10.120.0.0/14 + # SERVICE_CIDR=10.123.240.0/20 + # CLUSTER_MASTER=10.123.240.1 + # CLUSTER_DNS=10.123.240.10 + + KUBELET_OPTS="--address=10.64.33.81 \ + --allow-privileged=true \ + --api-servers=https://10.64.33.81:6443 \ + --cadvisor-port=4194 \ + --cert-dir=/srv/kubernetes \ + --cluster-dns=10.123.240.10 \ + --cluster-domain=cluster.local \ + --config=/etc/kubernetes/manifests \ + --configure-cbr0=false \ + --container-runtime=docker \ + --docker=unix:///var/run/docker.sock \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10248 \ + --hostname-override=10.64.33.81 \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master-service-namespace=default \ + --max-open-files=1000000 \ + --max-pods=110 \ + --node-ip=10.64.33.81 \ + --non-masquerade-cidr=10.0.0.0/8 \ + --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ + --pods-per-core=0 \ + --port=10250 \ + --read-only-port=10255 \ + --root-dir=/var/lib/kubelet \ + --tls-cert-file=/srv/kubernetes/server.cert \ + --tls-private-key-file=/srv/kubernetes/server.key \ + --v=2" + +Manually install + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/conf/kubelet /etc/sysconfig/ + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/system/kubelet.service /etc/systemd/system + + [vagrant@localhost ~]$ sudo systemctl enable kubelet.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /etc/systemd/system/kubelet.service. + +Experimental run + + [vagrant@localhost ~]$ KUBELET_OPTS="--address=0.0.0.0 \ + > --allow-privileged=true \ + > --api-servers=https://10.64.33.81:6443 \ + > --cadvisor-port=4194 \ + > --cert-dir=/srv/kubernetes \ + > --cluster-dns=10.123.240.10 \ + > --cluster-domain=cluster.local \ + > --config=/opt/kubernetes/manifests \ + > --configure-cbr0=false \ + > --container-runtime=docker \ + > --docker=unix:///var/run/docker.sock \ + > --experimental-nvidia-gpus=0 \ + > --healthz-bind-address=127.0.0.1 \ + > --healthz-port=10248 \ + > --hostname-override=10.64.33.81 \ + > --kubeconfig=/srv/kubernetes/kubeconfig \ + > --master-service-namespace=default \ + > --max-open-files=1000000 \ + > --max-pods=110 \ + > --node-ip=10.64.33.81 \ + > --non-masquerade-cidr=10.0.0.0/8 \ + > --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ + > --pods-per-core=0 \ + > --port=10250 \ + > --root-dir=/var/lib/kubelet \ + > --tls-cert-file=/srv/kubernetes/server.cert \ + > --tls-private-key-file=/srv/kubernetes/server.key \ + > --v=2"; sudo /opt/kubernetes/server/bin/kubelet $KUBELET_OPTS + +First start + + [vagrant@localhost ~]$ sudo systemctl daemon-reload + + [vagrant@localhost ~]$ sudo systemctl start kubelet.service + +Validation + + [vagrant@localhost ~]$ sudo systemctl -l status kubelet.service + 鈼▒ kubelet.service - Kubernetes container mgmt daemon - kubelet + Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled) + Active: active (running) since Sat 2016-11-12 15:11:08 UTC; 40min ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 17657 (kubelet) + CGroup: /system.slice/kubelet.service + 鈹溾攢17657 /opt/kubernetes/server/bin/kubelet --address=10.64.33.81 --allow-privileged=true --api-servers=https://10.64.33.81:6443 --cadvisor-port=4194 --cert-dir=/srv/kubernetes --cluster-dns=10.123.240.10 --cluster-domain=cluster.local --config=/etc/kubernetes/manifests --configure-cbr0=false --container-runtime=docker --docker=unix:///var/run/docker.sock --healthz-bind-address=127.0.0.1 --healthz-port=10248 --hostname-override=10.64.33.81 --kubeconfig=/srv/kubernetes/kubeconfig --master-service-namespace=default --max-open-files=1000000 --max-pods=0 --node-ip=10.64.33.81 --non-masquerade-cidr=10.0.0.0/8 --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 --pods-per-core=0 --port=10250 --read-only-port=10255 --root-dir=/var/lib/kubelet --tls-cert-file=/srv/kubernetes/server.cert --tls-private-key-file=/srv/kubernetes/server.key --v=2 + + Nov 12 15:46:09 localhost.localdomain kubelet[17657]: W1112 15:46:09.340116 17657 container_manager_linux.go:589] CPUAccounting not enabled for pid: 1076 + Nov 12 15:46:09 localhost.localdomain kubelet[17657]: W1112 15:46:09.340156 17657 container_manager_linux.go:592] MemoryAccounting not enabled for pid: 1076 + Nov 12 15:46:09 localhost.localdomain kubelet[17657]: I1112 15:46:09.340171 17657 container_manager_linux.go:290] Discovered runtime cgroups name: /system.slice/docker.service + Nov 12 15:46:09 localhost.localdomain kubelet[17657]: W1112 15:46:09.340318 17657 container_manager_linux.go:589] CPUAccounting not enabled for pid: 17657 + Nov 12 15:46:09 localhost.localdomain kubelet[17657]: W1112 15:46:09.340334 17657 container_manager_linux.go:592] MemoryAccounting not enabled for pid: 17657 + Nov 12 15:51:09 localhost.localdomain kubelet[17657]: W1112 15:51:09.347243 17657 container_manager_linux.go:589] CPUAccounting not enabled for pid: 1076 + Nov 12 15:51:09 localhost.localdomain kubelet[17657]: W1112 15:51:09.347277 17657 container_manager_linux.go:592] MemoryAccounting not enabled for pid: 1076 + Nov 12 15:51:09 localhost.localdomain kubelet[17657]: I1112 15:51:09.347292 17657 container_manager_linux.go:290] Discovered runtime cgroups name: /system.slice/docker.service + Nov 12 15:51:09 localhost.localdomain kubelet[17657]: W1112 15:51:09.354968 17657 container_manager_linux.go:589] CPUAccounting not enabled for pid: 17657 + Nov 12 15:51:09 localhost.localdomain kubelet[17657]: W1112 15:51:09.355003 17657 container_manager_linux.go:592] MemoryAccounting not enabled for pid: 17657 + + [vagrant@localhost ~]$ sudo journalctl --follow --no-tail --no-pager --pager-end -u kubelet.service + + [vagrant@localhost ~]$ sudo tail -100 /var/log/messages | grep kubelet + + [vagrant@localhost ~]$ sudo journalctl -k -f + +Using kubectl + + [vagrant@localhost ~]$ kubectl get nodes + NAME STATUS AGE + 10.64.33.81 Ready 26m + + [vagrant@localhost ~]$ kubectl get nodes/10.64.33.81 -o yaml + apiVersion: v1 + kind: Node + metadata: + annotations: + volumes.kubernetes.io/controller-managed-attach-detach: "true" + creationTimestamp: 2016-11-11T22:58:41Z + labels: + beta.kubernetes.io/arch: amd64 + beta.kubernetes.io/os: linux + kubernetes.io/hostname: 10.64.33.81 + name: 10.64.33.81 + resourceVersion: "103" + selfLink: /api/v1/nodes/10.64.33.81 + uid: 63c86df9-a862-11e6-8d5b-5254009fbdfd + spec: + externalID: 10.64.33.81 + status: + addresses: + - address: 10.64.33.81 + type: LegacyHostIP + - address: 10.64.33.81 + type: InternalIP + allocatable: + alpha.kubernetes.io/nvidia-gpu: "0" + cpu: "1" + memory: 2916372Ki + pods: "110" + capacity: + alpha.kubernetes.io/nvidia-gpu: "0" + cpu: "1" + memory: 2916372Ki + pods: "110" + conditions: + - lastHeartbeatTime: 2016-11-11T23:25:59Z + lastTransitionTime: 2016-11-11T23:22:08Z + message: kubelet has sufficient disk space available + reason: KubeletHasSufficientDisk + status: "False" + type: OutOfDisk + - lastHeartbeatTime: 2016-11-11T23:25:59Z + lastTransitionTime: 2016-11-11T22:58:41Z + message: kubelet has sufficient memory available + reason: KubeletHasSufficientMemory + status: "False" + type: MemoryPressure + - lastHeartbeatTime: 2016-11-11T23:25:59Z + lastTransitionTime: 2016-11-11T23:22:08Z + message: kubelet is posting ready status + reason: KubeletReady + status: "True" + type: Ready + daemonEndpoints: + kubeletEndpoint: + Port: 10250 + nodeInfo: + architecture: amd64 + bootID: b14335fa-f746-46c7-9517-44f1f9353cc5 + containerRuntimeVersion: docker://1.10.3 + kernelVersion: 3.10.0-327.36.1.el7.x86_64 + kubeProxyVersion: v1.3.10 + kubeletVersion: v1.3.10 + machineID: c1bde9cb97f44f9194d26b125a13dd44 + operatingSystem: linux + osImage: CentOS Linux 7 (Core) + systemUUID: C1BDE9CB-97F4-4F91-94D2-6B125A13DD44 + +Investigate logs + + E1112 03:13:18.667969 9980 kubelet.go:954] Image garbage collection failed: unable to find data for container / + + E1111 22:58:41.544101 4389 factory.go:291] devicemapper filesystem stats will not be reported: RHEL/Centos 7.x kernel version 3.10.0-366 or later is required to use thin_ls - you have "3.10.0-327.36.1.el7.x86_64" + +* The kube-proxy + +Service file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/system/kube-proxy.service + [Unit] + Description=Kubernetes proxy networking + Documentation=https://github.com/kubernetes/kubernetes + # After= + # Requires= + # Wants= + # Conflicts=openshift-node.service + + [Service] + # Type=notify + # NotifyAccess=none + Type=simple + User=root + + Environment=KUBERNETES_RELEASE_VERSION=1.3.10 + EnvironmentFile=/etc/sysconfig/kube-proxy + WorkingDirectory=/opt/kubernetes + + ExecStart=/opt/kubernetes/server/bin/kube-proxy $KUBE_PROXY_OPTS + # ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_RELEASE_VERSION}/centos/bin/kube-proxy ${KUBE_PROXY_OPTS}" + + # Restart=always + Restart=on-failure + + [Install] + WantedBy=multi-user.target + +Opts file + + [vagrant@localhost ~]$ vi kubernetes/1.3.10/centos/systemd/conf/kube-proxy + # KUBE_PROXY_VERSION=1.3.10 + + # BIND_ADDRESS=10.64.33.81 + # HOSTNAME_OVERRIDE=10.64.33.81 + + # --iptables-masquerade-bit=14: If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within [0, 31] + + # KUBECONFIG_PATH=/srv/kubernetes/kubeconfig + # MASTER_ADDRESS=https://10.64.33.81:6443 + + # --service-cluster-ip-range=: A CIDR notation IP range + # Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 + # CLUSTER_CIDR=10.120.0.0/14 + # SERVICE_CIDR=10.123.240.0/20 + # CLUSTER_MASTER=10.123.240.1 + # CLUSTER_DNS=10.123.240.10 + + KUBE_PROXY_OPTS="--bind-address=10.64.33.81 \ + --cluster-cidr=10.120.0.0/14 \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10249 \ + --hostname-override=10.64.33.81 \ + --iptables-masquerade-bit=14 \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --v=2" + +Manually Install + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/conf/kube-proxy /etc/sysconfig/ + + [vagrant@localhost ~]$ sudo cp kubernetes/1.3.10/centos/systemd/system/kube-proxy.service /etc/systemd/system + + [vagrant@localhost ~]$ sudo systemctl enable kube-proxy.service + Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /etc/systemd/system/kube-proxy.service. + +Experimental Run + + [vagrant@localhost ~]$ KUBE_PROXY_OPTS="--bind-address=0.0.0.0 \ + > --cluster-cidr=10.120.0.0/14 \ + > --healthz-bind-address=127.0.0.1 \ + > --healthz-port=10249 \ + > --hostname-override=10.64.33.81 \ + > --iptables-masquerade-bit=14 \ + > --kubeconfig=/srv/kubernetes/kubeconfig \ + > --master=https://10.64.33.81:6443 \ + > --v=2"; sudo /opt/kubernetes/server/bin/kube-proxy $KUBE_PROXY_OPTS + +First start + + [vagrant@localhost ~]$ sudo systemctl daemon-reload + + [vagrant@localhost ~]$ sudo systemctl start kube-proxy.service + +Validation + + [vagrant@localhost ~]$ sudo systemctl -l status kube-proxy.service + 鈼▒ kube-proxy.service - Kubernetes proxy networking + Loaded: loaded (/etc/systemd/system/kube-proxy.service; enabled; vendor preset: disabled) + Active: active (running) since Sat 2016-11-12 15:47:53 UTC; 8min ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 20694 (kube-proxy) + CGroup: /system.slice/kube-proxy.service + 鈹斺攢20694 /opt/kubernetes/server/bin/kube-proxy --bind-address=10.64.33.81 --cluster-cidr=10.120.0.0/14 --healthz-bind-address=127.0.0.1 --healthz-port=10249 --hostname-override=10.64.33.81 --iptables-masquerade-bit=14 --kubeconfig=/srv/kubernetes/kubeconfig --master=https://10.64.33.81:6443 --v=2 + + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.916429 20694 server.go:155] setting OOM scores is unsupported in this build + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.948369 20694 server.go:202] Using iptables Proxier. + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.949826 20694 proxier.go:216] missing br-netfilter module or unset br-nf-call-iptables; proxy may not work as intended + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.949855 20694 server.go:214] Tearing down userspace rules. + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.959059 20694 conntrack.go:40] Setting nf_conntrack_max to 65536 + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.959696 20694 conntrack.go:57] Setting conntrack hashsize to 16384 + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.959785 20694 conntrack.go:62] Setting nf_conntrack_tcp_timeout_established to 86400 + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.961241 20694 proxier.go:440] Adding new service "default/kubernetes:https" at 10.123.240.1:443/TCP + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.961333 20694 proxier.go:674] Not syncing iptables until Services and Endpoints have been received from master + Nov 12 15:47:53 localhost.localdomain kube-proxy[20694]: I1112 15:47:53.977742 20694 proxier.go:516] Setting endpoints for "default/kubernetes:https" to [10.64.33.81:6443] + + [vagrant@localhost ~]$ sudo journalctl --follow --no-tail --no-pager --pager-end -u kube-proxy.service + +Using kubectl + + [vagrant@localhost ~]$ kubectl get endpoints,services + NAME ENDPOINTS AGE + kubernetes 10.64.33.81:6443 1d + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kubernetes 10.123.240.1 443/TCP 1d + +### Run kub-apiserver, kube-controller-manager, kube-scheduler, kube-proxy as PODs + +Reference + + https://github.com/coreos/coreos-kubernetes/blob/v0.4.0/multi-node/generic/controller-install.sh + +POD image and container +------------------------ + +Save local pulled + + [tangfx@localhost ~]$ docker save -o gcr.io%252Fgoogle_containers%252Fpause-amd64%253A3.0.tar 99e59f495ffa + +A *gofileserver* HTTP archives + + tangf@DESKTOP-H68OQDV /cygdrive/g/2015-12-19-repository/99-mirror + $ gofileserver.exe + Listening at :48080 + + [vagrant@localhost ~]$ curl -L http://192.168.1.100:48080/ + 
+    centos/
+    gcr.io%2Fgoogle_containers/
+
+ + [vagrant@localhost ~]$ wget -r http://192.168.1.100:48080/gcr.io%252Fgoogle_containers/ + --2016-11-12 18:55:01-- http://192.168.1.100:48080/gcr.io%252Fgoogle_containers/gcr.io%252Fgoogle_containers%252Fkubernetes-dashboard-amd64%253Av1.0.1.tar + Reusing existing connection to 192.168.1.100:48080. + HTTP request sent, awaiting response... 200 OK + Length: 44133888 (42M) [application/x-tar] + Saving to: 鈥▒192.168.1.100:48080/gcr.io%2Fgoogle_containers/gcr.io%2Fgoogle_containers%2Fkubernetes-dashboard-amd64%3Av1.0.1.tar鈥▒ + + 30% [==========> ] 13,531,220 33.2KB/s eta 10m 40s + +Load from image archive + + [vagrant@localhost ~]$ mv gcr.io%252Fgoogle_containers%252Fpause-amd64%253A3.0.tar pause-amd64.tar + + [vagrant@localhost ~]$ docker load -i pause-amd64.tar + + [vagrant@localhost ~]$ docker images + 99e59f495ffa 6 months ago 746.9 kB + + [vagrant@localhost ~]$ docker tag 99e59f495ffa gcr.io/google_containers/pause-amd64:3.0 + +Validation + + [vagrant@localhost addons]$ kubectl run netcat-hello-http --image=tangfeixiong/netcat-hello-http + + [vagrant@localhost addons]$ kubectl get deployments,pods + NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + netcat-hello-http 1 1 1 1 7h + NAME READY STATUS RESTARTS AGE + netcat-hello-http-839099240-o5q24 1/1 Running 0 24m + + [vagrant@localhost addons]$ kubectl expose deployments/netcat-hello-http --port=80 + service "netcat-hello-http" exposed + + [vagrant@localhost addons]$ kubectl get ep,svc + NAME ENDPOINTS AGE + kubernetes 10.64.33.81:6443 2d + netcat-hello-http 10.121.24.2:80 1m + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kubernetes 10.123.240.1 443/TCP 2d + netcat-hello-http 10.123.249.219 80/TCP 1m + + [vagrant@localhost ~]$ curl 10.121.24.2 + welcome

hello world

+ + [vagrant@localhost ~]$ curl 10.123.249.219 + welcome

hello world

+ + [tangfx@localhost ~]$ curl 10.121.24.2 + welcome

hello world

+ + [tangfx@localhost ~]$ curl 10.123.249.219 + welcome

hello world

+ +Kubernetes add-ons +-------------------- + +* Start kube-dns + +Using v19 + + [vagrant@localhost addons]$ kubectl create -f kube-dns-rc.yaml + replicationController "kube-dns-v19" created + + [vagrant@localhost addons]$ kubectl get rc,pod --namespace=kube-system + NAME DESIRED CURRENT AGE + kube-dns-v19 1 1 3m + NAME READY STATUS RESTARTS AGE + kube-dns-v19-o32di 2/3 Running 2 3m + + [vagrant@localhost addons]$ kubectl logs pods/kube-dns-v19-o32di -c=kubedns --namespace=kube-system + + [vagrant@localhost addons]$ kubectl logs pods/kube-dns-v19-o32di -c=dnsmasq --namespace=kube-system + + [vagrant@localhost addons]$ kubectl logs pods/kube-dns-v19-o32di -c=healthz --namespace=kube-system + + [vagrant@localhost addons]$ kubectl create -f kube-dns-svc.yaml + service "kube-dns" created + + [vagrant@localhost addons]$ kubectl get ep,svc --namespace=kube-system + NAME ENDPOINTS AGE + kube-dns 6m + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kube-dns 10.123.240.10 53/UDP,53/TCP 6m + +* Start kubernetes-dashboard + + [vagrant@localhost addons]$ kubectl create -f kube-dashboard-rc.yaml + replicationcontroller "kubernetes-dashboard-v1.4.0" created + + [vagrant@localhost addons]$ kubectl get pods,replicationcontrollers --namespace=kube-system + + [vagrant@localhost addons]$ kubectl create -f kube-dashboard-svc.yaml + service "kubernetes-dashboard" created + + [vagrant@localhost addons]$ kubectl get ep,svc --namespace=kube-system + NAME ENDPOINTS AGE + kube-dns 1h + kubernetes-dashboard 10.121.24.4:9090 5m + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kube-dns 10.123.240.10 53/UDP,53/TCP 1h + kubernetes-dashboard 10.123.243.229 80/TCP 3m + +* Issue + + https://github.com/kubernetes/kubernetes/issues/34379 + + [vagrant@localhost addons]$ sudo systemctl stop kube-apiserver.service + + [vagrant@localhost addons]$ etcdctl ls /registry/secrets/default + /registry/secrets/default/default-token-84bco + + [vagrant@localhost addons]$ etcdctl ls /registry/secrets/kube-system + /registry/secrets/kube-system/default-token-g37yb + + [vagrant@localhost addons]$ etcdctl rm /registry/secrets/default/default-token-84bco + + [vagrant@localhost addons]$ etcdctl rm /registry/secrets/kube-system/default-token-g37yb + + [vagrant@localhost addons]$ sudo systemctl start kube-apiserver.service + +Install worker node +-------------------- + +Same as preview steps, only validate to: + +* copy `CA and certs` + +* install `kubelet` + +* install `kube-proxy` + +* load `POD image` + +* Validation + +After `docker` and `flanneld` installed + +Copy `CA and certs` + + [tangfx@localhost ~]$ sudo cp .pki/kubernetes/* /etc/kubernetes/cacerts + +Install `kubelet` + + [tangfx@localhost ~]$ vi kubernetes/1.3.1/fedora23/systemd/system/kubelet.service + [Unit] + Description=Kubernetes container mgmt daemon - kubelet + Documentation=https://github.com/kubernetes/kubernetes + After=docker.service + Requires=docker.service + # Wants= + Conflicts=openshift-node.service + + [Service] + Type=simple + # Type=notify + # NotifyAccess=all + User=root + + WorkingDirectory=/opt/kubernetes + + Environment=KUBERNETES_RELEASE_VERSION=1.3.1 + EnvironmentFile=/etc/sysconfig/kubelet + + ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/usr/bin/mkdir -p /var/log/containers + + # ExecStart=/opt/kubernetes/server/bin/kubelet $KUBELET_OPTS + ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_RELEASE_VERSION}/fedora23/bin/kubelet ${KUBELET_OPTS}" + + # Restart=always + Restart=on-failure + RestartSec=10 + KillMode=process + + [Install] + WantedBy=multi-user.target + + [tangfx@localhost ~]$ vi kubernetes/1.3.1/fedora23/systemd/conf/kubelet + # KUBELET_VERSION=1.3.10 + + # --api-servers=[]: Comma separated list of Kubernetes API servers (ip:port). + # API_SERVERS=https://10.64.33.81:6443 + + # --cert-dir="/var/run/kubernetes" + # CERT_DIR=/home/tangfx/.pki/kubernetes + + # --hostname-override="": As identification instad of the actual hostname + # HOSTNAME_OVERRIDE=10.64.33.90 + + # --kubeconfig="/var/lib/kubelet/kubeconfig": Path to a kubeconfig file + # KUBECONFIG_PATH=/home/tangfx/.pki/kubernetes/kubeconfig + + # --master-service-namespace="default": The master services should be injected into pods + + # --service-cluster-ip-range=: A CIDR notation IP range + # Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 + # SERVICE_CIDR=10.123.240.0/20 + # SERVICE_MASTER=10.123.240.1 + # CLUSTER_DNS=10.123.240.10 + + # TLS_SERVER_CERT=/home/tangfx/.pki/kubernetes/server.cert + # TLS_SERVER_KEY=/home/tangfx/.pki/kubernetes/server.key + + #KUBELET_OPTS="--address=10.64.33.90 \ + # --allow-privileged=true \ + # --api-servers=https://10.64.33.81:6443 \ + # --cadvisor-port=4194 \ + # --cert-dir=/home/tangfx/.pki/kubernetes \ + # --cluster-dns=10.123.240.10 \ + # --cluster-domain=cluster.local \ + # --config=/etc/kubernetes/manifests \ + # --configure-cbr0=false \ + # --container-runtime=docker \ + # --docker=unix:///var/run/docker.sock \ + # --healthz-bind-address=127.0.0.1 \ + # --healthz-port=10248 \ + # --hostname-override=10.64.33.90 \ + # --kubeconfig=/home/tangfx/.pki/kubernetes/kubeconfig \ + # --master-service-namespace=default \ + # --max-open-files=1000000 \ + # --max-pods=110 \ + # --node-ip=10.64.33.90 \ + # --non-masquerade-cidr=10.0.0.0/8 \ + # --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ + # --pods-per-core=0 \ + # --port=10250 \ + # --read-only-port=10255 \ + # --root-dir=/var/lib/kubelet \ + # --tls-cert-file=/home/tangfx/.pki/kubernetes/server.cert \ + # --tls-private-key-file=/home/tangfx/.pki/kubernetes/server.key \ + # --v=2" + + KUBELET_OPTS="--address=10.64.33.90 \ + --allow-privileged=false \ + --api-servers=https://10.64.33.81:6443 \ + --cluster-dns=10.123.240.10 \ + --cluster-domain=cluster.local \ + --config=/etc/kubernetes/manifests \ + --hostname-override=10.64.33.90 \ + --kubeconfig=/home/tangfx/.pki/kubernetes/kubeconfig \ + --node-ip=10.64.33.90 \ + --v=2" + + [tangfx@localhost ~]$ sudo cp kubernetes/1.3.1/fedora23/systemd/system/kubelet.service /etc/systemd/system/ + + [tangfx@localhost ~]$ sudo cp kubernetes/1.3.1/fedora23/systemd/conf/kubelet /etc/sysconfig/ + + [tangfx@localhost ~]$ sudo systemctl enable kubelet.service + + [tangfx@localhost ~]$ sudo systemctl daemon-reload + + [tangfx@localhost ~]$ sudo systemctl start kubelet.service + + [tangfx@localhost ~]$ sudo systemctl -l status kubelet.service + 鈼▒ kubelet.service - Kubernetes container mgmt daemon - kubelet + Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled) + Active: active (running) since Sat 2016-11-12 23:39:31 CST; 3h 52min ago + Docs: https://github.com/kubernetes/kubernetes + Process: 10194 ExecStartPre=/usr/bin/mkdir -p /var/log/containers (code=exited, status=0/SUCCESS) + Process: 10192 ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests (code=exited, status=0/SUCCESS) + Main PID: 10196 (kubelet) + CGroup: /system.slice/kubelet.service + 鈹溾攢10196 /opt/kubernetes/1.3.1/fedora23/bin/kubelet --address=10.64.33.90 --allow-privileged=false --api-servers=https://10.64.33.81:6443 --cluster-dns=10.123.240.10 --cluster-domain=cluster.local --config=/etc/kubernetes/manifests --hostname-override=10.64.33.90 --kubeconfig=/home/tangfx/.pki/kubernetes/kubeconfig --node-ip=10.64.33.90 --v=2 + 鈹斺攢10215 journalctl -k -f + + Nov 13 03:29:32 localhost.localdomain bash[10196]: W1113 03:29:32.995284 10196 container_manager_linux.go:573] CPUAccounting not enabled for pid: 10196 + Nov 13 03:29:32 localhost.localdomain bash[10196]: W1113 03:29:32.995567 10196 container_manager_linux.go:576] MemoryAccounting not enabled for pid: 10196 + Nov 13 03:29:40 localhost.localdomain bash[10196]: I1113 03:29:40.967632 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + Nov 13 03:29:56 localhost.localdomain bash[10196]: I1113 03:29:56.082104 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + Nov 13 03:30:11 localhost.localdomain bash[10196]: I1113 03:30:11.241585 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + Nov 13 03:30:26 localhost.localdomain bash[10196]: I1113 03:30:26.333483 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + Nov 13 03:30:41 localhost.localdomain bash[10196]: I1113 03:30:41.507199 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + Nov 13 03:30:56 localhost.localdomain bash[10196]: I1113 03:30:56.594531 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + Nov 13 03:31:11 localhost.localdomain bash[10196]: I1113 03:31:11.687486 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + Nov 13 03:31:26 localhost.localdomain bash[10196]: I1113 03:31:26.816454 10196 thin_pool_watcher.go:126] reserving metadata snapshot for thin-pool docker-253:0-67108972-pool + + [tangfx@localhost ~]$ sudo journalctl --no-tail --no-pager --pager-end --lines=10 --unit=kubelet.service + +Install `kube-proxy` + + [tangfx@localhost ~]$ vi kubernetes/1.3.1/fedora23/systemd/system/kube-proxy.service + [Unit] + Description=Kubernetes proxy networking + Documentation=https://github.com/kubernetes/kubernetes + # After= + # Requires= + # Wants= + # Conflicts=openshift-node.service + + [Service] + # Type=notify + # NotifyAccess=none + Type=simple + User=root + + Environment=KUBERNETES_RELEASE_VERSION=1.3.1 + EnvironmentFile=/etc/sysconfig/kube-proxy + WorkingDirectory=/opt/kubernetes + + # ExecStart=/opt/kubernetes/server/bin/kube-proxy $KUBE_PROXY_OPTS + ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_RELEASE_VERSION}/fedora23/bin/kube-proxy ${KUBE_PROXY_OPTS}" + + # Restart=always + Restart=on-failure + + [Install] + WantedBy=multi-user.target + + [tangfx@localhost ~]$ vi kubernetes/1.3.1/fedora23/systemd/conf/kube-proxy + # KUBE_PROXY_VERSION=1.3.10 + + # BIND_ADDRESS=10.64.33.90 + # HOSTNAME_OVERRIDE=10.64.33.90 + + # --iptables-masquerade-bit=14: If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within [0, 31] + + # KUBECONFIG_PATH=/home/tangfx/.pki/kubernetes/kubeconfig + + # MASTER_ADDRESS=https://10.64.33.81:6443 + + # --service-cluster-ip-range=: A CIDR notation IP range + # Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 + # CLUSTER_CIDR=10.120.0.0/14 + # SERVICE_CIDR=10.123.240.0/20 + # CLUSTER_MASTER=10.123.240.1 + # SERVICE_DNS=10.123.240.10 + + KUBE_PROXY_OPTS="--bind-address=10.64.33.90 \ + --cluster-cidr=10.120.0.0/14 \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10249 \ + --hostname-override=10.64.33.90 \ + --iptables-masquerade-bit=14 \ + --kubeconfig=/home/tangfx/.pki/kubernetes/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --v=2" + + [tangfx@localhost ~]$ sudo cp kubernetes/1.3.1/fedora23/systemd/system/kube-proxy.service /etc/systemd/system/ + + [tangfx@localhost ~]$ sudo cp kubernetes/1.3.1/fedora23/systemd/conf/kube-proxy /etc/sysconfig/ + + [tangfx@localhost ~]$ sudo systemctl enable kube-proxy.service + + [tangfx@localhost ~]$ sudo systemctl daemon-reload + + [tangfx@localhost ~]$ sudo systemctl start kube-proxy.service + + [tangfx@localhost ~]$ sudo systemctl -l status kube-proxy.service + 鈼▒ kube-proxy.service - Kubernetes proxy networking + Loaded: loaded (/etc/systemd/system/kube-proxy.service; enabled; vendor preset: disabled) + Active: active (running) since Sat 2016-11-12 23:39:58 CST; 3h 42min ago + Docs: https://github.com/kubernetes/kubernetes + Main PID: 10262 (kube-proxy) + CGroup: /system.slice/kube-proxy.service + 鈹斺攢10262 /opt/kubernetes/1.3.1/fedora23/bin/kube-proxy --bind-address=10.64.33.90 --cluster-cidr=10.120.0.0/14 --healthz-bind-address=127.0.0.1 --healthz-port=10249 --hostname-override=10.64.33.90 --iptables-masquerade-bit=14 --kubeconfig=/home/tangfx/.pki/kubernetes/kubeconfig --master=https://10.64.33.81:6443 --v=2 + + Nov 12 23:39:58 localhost.localdomain bash[10262]: I1112 23:39:58.626897 10262 conntrack.go:36] Setting nf_conntrack_max to 262144 + Nov 12 23:39:58 localhost.localdomain bash[10262]: I1112 23:39:58.627147 10262 conntrack.go:41] Setting conntrack hashsize to 65536 + Nov 12 23:39:58 localhost.localdomain bash[10262]: I1112 23:39:58.627751 10262 conntrack.go:46] Setting nf_conntrack_tcp_timeout_established to 86400 + Nov 12 23:39:58 localhost.localdomain bash[10262]: I1112 23:39:58.631191 10262 proxier.go:427] Adding new service "default/kubernetes:https" at 10.123.240.1:443/TCP + Nov 12 23:39:58 localhost.localdomain bash[10262]: I1112 23:39:58.632505 10262 proxier.go:647] Not syncing iptables until Services and Endpoints have been received from master + Nov 12 23:39:58 localhost.localdomain bash[10262]: I1112 23:39:58.641024 10262 proxier.go:502] Setting endpoints for "default/kubernetes:https" to [10.64.33.81:6443] + + [tangfx@localhost ~]$ sudo journalctl --no-tail --no-pager --pager-end --lines=10 --unit=kube-proxy.service + +Load `POD image` + + [tangfx@localhost ~]$ curl http://192.168.1.100:48080 + 
+    centos/
+    gcr.io%2Fgoogle_containers/
+
+ + [tangfx@localhost ~]$ curl http://192.168.1.100:48080/gcr.io%252Fgoogle_containers/ + 
+    gcr.io%2Fgoogle_containers%2Fkubernetes-dashboard-amd64%3Av1.0.1.tar
+    gcr.io%2Fgoogle_containers%2Fkubernetes-dashboard-amd64%3Av1.1.1.tar
+    gcr.io%2Fgoogle_containers%2Fkubernetes-dashboard-amd64%3Av1.4.0.tar
+    gcr.io%2Fgoogle_containers%2Fkubernetes-dashboard-amd64%3Av1.4.2.tar
+    gcr.io%2Fgoogle_containers%2Fpause%3A0.8.0.tar
+    gcr.io%2Fgoogle_containers%2Fpause%3A2.0.tar
+    gcr.io%2Fgoogle_containers%2Fpause-amd64%3A3.0.tar
+
+ + [tangfx@localhost ~]$ curl http://192.168.1.100:48080/gcr.io%252Fgoogle_containers/gcr.io%252Fgoogle_containers%252Fpause-amd64%253A3.0.tar -O + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 100 745k 100 745k 0 0 34716 0 0:00:21 0:00:21 --:--:-- 54639 + + [tangfx@localhost ~]$ docker load -i gcr.io%252Fgoogle_containers%252Fpause-amd64%253A3.0.tar + +Validation + + [tangfx@localhost ~]$ KUBECONFIG=.pki/kubernetes/kubeconfig kubectl get nodes + NAME STATUS AGE + 10.64.33.81 Ready 20h + 10.64.33.90 Ready 12h + +Scale + + [tangfx@localhost ~]$ KUBECONFIG=.pki/kubernetes/kubeconfig kubectl scale --replicas=2 deployment/netcat-hello-http + + [tangfx@localhost ~]$ KUBECONFIG=.pki/kubernetes/kubeconfig kubectl get ep,svc + NAME ENDPOINTS AGE + ep/kubernetes 10.64.33.81:6443 1d + ep/netcat-hello-http 10.120.22.2:80,10.121.24.2:80 1d + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + svc/kubernetes 10.123.240.1 443/TCP 1d + svc/netcat-hello-http 10.123.242.12 80/TCP 1d + + [tangfx@localhost ~]$ curl 10.120.22.2 + welcome

hello world

+ + [tangfx@localhost ~]$ curl 10.121.24.2 + welcome

hello world

+ + [tangfx@localhost ~]$ curl 10.123.242.12 + welcome

hello world

diff --git a/hack/kubernetes-clearwater/kubernetes/1.1.2/manifests/addons/kube-dns-rc.json b/hack/kubernetes-clearwater/kubernetes/1.1.2/manifests/addons/kube-dns-rc.json new file mode 100755 index 0000000..456ebfa --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.1.2/manifests/addons/kube-dns-rc.json @@ -0,0 +1,146 @@ +{ + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "version": "v9" + }, + "name": "kube-dns-v9", + "namespace": "kube-system" + }, + "spec": { + "replicas": 1, + "selector": { + "k8s-app": "kube-dns", + "version": "v9" + }, + "template": { + "metadata": { + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "version": "v9" + } + }, + "spec": { + "containers": [ + { + "command": [ + "/usr/local/bin/etcd", + "-data-dir", + "/var/etcd/data", + "-listen-client-urls", + "http://127.0.0.1:2379,http://127.0.0.1:4001", + "-advertise-client-urls", + "http://127.0.0.1:2379,http://127.0.0.1:4001", + "-initial-cluster-token", + "skydns-etcd" + ], + "image": "gcr.io/google_containers/etcd:2.0.9", + "name": "etcd", + "resources": { + "limits": { + "cpu": "100m", + "memory": "50Mi" + } + }, + "volumeMounts": [ + { + "mountPath": "/var/etcd/data", + "name": "etcd-storage" + } + ] + }, + { + "args": [ + "-domain=cluster.local" + ], + "image": "gcr.io/google_containers/kube2sky:1.11", + "name": "kube2sky", + "resources": { + "limits": { + "cpu": "100m", + "memory": "50Mi" + } + } + }, + { + "args": [ + "-machines=http://127.0.0.1:4001", + "-addr=0.0.0.0:53", + "-ns-rotate=false", + "-domain=cluster.local." + ], + "image": "gcr.io/google_containers/skydns:2015-10-13-8c72f8c", + "livenessProbe": { + "httpGet": { + "path": "/healthz", + "port": 8080, + "scheme": "HTTP" + }, + "initialDelaySeconds": 30, + "timeoutSeconds": 5 + }, + "name": "skydns", + "ports": [ + { + "containerPort": 53, + "name": "dns", + "protocol": "UDP" + }, + { + "containerPort": 53, + "name": "dns-tcp", + "protocol": "TCP" + } + ], + "readinessProbe": { + "httpGet": { + "path": "/healthz", + "port": 8080, + "scheme": "HTTP" + }, + "initialDelaySeconds": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "cpu": "100m", + "memory": "50Mi" + } + } + }, + { + "args": [ + "-cmd=nslookup kubernetes.default.svc.cluster.local localhost >/dev/null", + "-port=8080" + ], + "image": "gcr.io/google_containers/exechealthz:1.0", + "name": "healthz", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "cpu": "10m", + "memory": "20Mi" + } + } + } + ], + "dnsPolicy": "Default", + "volumes": [ + { + "emptyDir": {}, + "name": "etcd-storage" + } + ] + } + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.1.2/manifests/addons/kube-dns-svc.json b/hack/kubernetes-clearwater/kubernetes/1.1.2/manifests/addons/kube-dns-svc.json new file mode 100755 index 0000000..eaa8176 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.1.2/manifests/addons/kube-dns-svc.json @@ -0,0 +1,31 @@ +{ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "kube-dns", + "namespace": "kube-system", + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/name": "KubeDNS", + "kubernetes.io/cluster-service": "true" + } + }, + "spec": { + "clusterIP": "10.123.240.10", + "ports": [ + { + "protocol": "UDP", + "name": "dns", + "port": 53 + }, + { + "protocol": "TCP", + "name": "dns-tcp", + "port": 53 + } + ], + "selector": { + "k8s-app": "kube-dns" + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.2.4/manifests/addons/kube-dns-rc.json b/hack/kubernetes-clearwater/kubernetes/1.2.4/manifests/addons/kube-dns-rc.json new file mode 100755 index 0000000..03aa9c0 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.2.4/manifests/addons/kube-dns-rc.json @@ -0,0 +1,164 @@ +{ + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "version": "v11" + }, + "name": "kube-dns-v11", + "namespace": "kube-system" + }, + "spec": { + "replicas": 1, + "selector": { + "k8s-app": "kube-dns", + "version": "v11" + }, + "template": { + "metadata": { + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "version": "v11" + } + }, + "spec": { + "containers": [ + { + "command": [ + "/usr/local/bin/etcd", + "-data-dir", + "/var/etcd/data", + "-listen-client-urls", + "http://127.0.0.1:2379,http://127.0.0.1:4001", + "-advertise-client-urls", + "http://127.0.0.1:2379,http://127.0.0.1:4001", + "-initial-cluster-token", + "skydns-etcd" + ], + "image": "gcr.io/google_containers/etcd-amd64:2.2.1", + "name": "etcd", + "resources": { + "limits": { + "cpu": "100m", + "memory": "500Mi" + }, + "requests": { + "cpu": "100m", + "memory": "50Mi" + } + }, + "volumeMounts": [ + { + "mountPath": "/var/etcd/data", + "name": "etcd-storage" + } + ] + }, + { + "args": [ + "--domain=cluster.local" + ], + "image": "gcr.io/google_containers/kube2sky:1.14", + "livenessProbe": { + "failureThreshold": 5, + "httpGet": { + "path": "/healthz", + "port": 8080, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "name": "kube2sky", + "readinessProbe": { + "httpGet": { + "path": "/readiness", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 30, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "cpu": "100m", + "memory": "200Mi" + }, + "requests": { + "cpu": "100m", + "memory": "50Mi" + } + } + }, + { + "args": [ + "-machines=http://127.0.0.1:4001", + "-addr=0.0.0.0:53", + "-ns-rotate=false", + "-domain=cluster.local." + ], + "image": "gcr.io/google_containers/skydns:2015-10-13-8c72f8c", + "name": "skydns", + "ports": [ + { + "containerPort": 53, + "name": "dns", + "protocol": "UDP" + }, + { + "containerPort": 53, + "name": "dns-tcp", + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "cpu": "100m", + "memory": "200Mi" + }, + "requests": { + "cpu": "100m", + "memory": "50Mi" + } + } + }, + { + "args": [ + "-cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null", + "-port=8080" + ], + "image": "gcr.io/google_containers/exechealthz:1.0", + "name": "healthz", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "cpu": "10m", + "memory": "20Mi" + }, + "requests": { + "cpu": "10m", + "memory": "20Mi" + } + } + } + ], + "dnsPolicy": "Default", + "volumes": [ + { + "emptyDir": {}, + "name": "etcd-storage" + } + ] + } + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.2.4/manifests/addons/kube-dns-svc.json b/hack/kubernetes-clearwater/kubernetes/1.2.4/manifests/addons/kube-dns-svc.json new file mode 100755 index 0000000..eaa8176 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.2.4/manifests/addons/kube-dns-svc.json @@ -0,0 +1,31 @@ +{ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "kube-dns", + "namespace": "kube-system", + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/name": "KubeDNS", + "kubernetes.io/cluster-service": "true" + } + }, + "spec": { + "clusterIP": "10.123.240.10", + "ports": [ + { + "protocol": "UDP", + "name": "dns", + "port": 53 + }, + { + "protocol": "TCP", + "name": "dns-tcp", + "port": 53 + } + ], + "selector": { + "k8s-app": "kube-dns" + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-apiserver b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-apiserver new file mode 100644 index 0000000..adea869 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-apiserver @@ -0,0 +1,57 @@ +# KUBE_APISERVER_VERSION=1.3.10 + +# --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota + +# --advertise-address=: IP address to advertise the apiserver to members of the cluster, must be reachable by the cluster +# ADVERTISE_APISERVER=10.64.33.81 + +# --cert-dir="/var/run/kubernetes" +# CERT_DIR=/srv/kubernetes + +# --enable-swagger-ui[=false]: Enables swagger ui at /swagger-ui +# ENABLE_SWAGGER_UI=false + +# --etcd-servers=: comma separated etcd servers to connect (http://ip:port) +# ETCD_SERVERS=http://10.64.33.81:2379 + +# --master-service-namespace="default": in which the kubernetes master services should be injected into pods + +# --runtime-config=: A set of key=value pairs such as apis/=true/false, also apis// +# RUNTIME_CONFIG=api/all=true + +# --secure-port=6443: The port on which to serve HTTPS +# SECURE_PORT=6443 + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +# --service-node-port-range=: default '30000-32767' + +KUBE_APISERVER_OPTS="--admission-control=AlwaysAdmit \ + --advertise-address=10.64.33.81 \ + --allow-privileged=true \ + --apiserver-count=1 \ + --bind-address=0.0.0.0 \ + --cert-dir=/srv/kubernetes \ + --client-ca-file=/srv/kubernetes/ca.crt \ + --enable-garbage-collector=false \ + --enable-swagger-ui=false \ + --etcd-servers=http://10.64.33.81:2379 \ + --insecure-bind-address=127.0.0.1 \ + --insecure-port=8080 \ + --kubelet-certificate-authority=/srv/kubernetes/ca.crt \ + --kubelet-client-certificate=/srv/kubernetes/kubecfg.crt \ + --kubelet-client-key=/srv/kubernetes/kubecfg.key \ + --kubelet-https=true \ + --master-service-namespace=default \ + --runtime-config=api/all=true \ + --secure-port=6443 \ + --service-account-key-file=/srv/kubernetes/server.key \ + --service-cluster-ip-range=10.123.240.0/20 \ + --tls-cert-file=/srv/kubernetes/server.cert \ + --tls-private-key-file=/srv/kubernetes/server.key \ + --v=2" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-controller-manager b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-controller-manager new file mode 100644 index 0000000..7a53b36 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-controller-manager @@ -0,0 +1,28 @@ +# KUBE_CONTROLLER_MANAGER_VERSION=1.3.10 + +# ADDRESS_LISTEN=10.64.33.81 + +# --cloud-config="": The path to the cloud provider configuration file + +# --cloud-provider="": The provider for cloud services + +# MASTER_ADDRESS=https://10.64.33.81:6443 + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBE_CONTROLLER_MANAGER_OPTS="--address=10.64.33.81 \ + --cluster-cidr=10.120.0.0/14 \ + --cluster-name=kubernetes \ + --enable-garbage-collector=false \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --port=10252 \ + --root-ca-file=/srv/kubernetes/ca.crt \ + --service-account-private-key-file=/srv/kubernetes/server.key \ + --service-cluster-ip-range=10.123.240.0/20 \ + --v=2" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-proxy b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-proxy new file mode 100644 index 0000000..a90115f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-proxy @@ -0,0 +1,26 @@ +# KUBE_PROXY_VERSION=1.3.10 + +# BIND_ADDRESS=10.64.33.81 +# HOSTNAME_OVERRIDE=10.64.33.81 + +# --iptables-masquerade-bit=14: If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within [0, 31] + +# KUBECONFIG_PATH=/srv/kubernetes/kubeconfig +# MASTER_ADDRESS=https://10.64.33.81:6443 + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBE_PROXY_OPTS="--bind-address=10.64.33.81 \ + --cluster-cidr=10.120.0.0/14 \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10249 \ + --hostname-override=10.64.33.81 \ + --iptables-masquerade-bit=14 \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --v=2" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-scheduler b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-scheduler new file mode 100644 index 0000000..f21561c --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kube-scheduler @@ -0,0 +1,22 @@ +# KUBE_SCHEDULER_VERSION=1.3.10 + +# ADDRESS_LISTEN=10.64.33.81 +# MASTER_ADDRESS=https://10.64.33.81:6443 + +# --cloud-config="": The path to the cloud provider configuration file +# --cloud-provider="": The provider for cloud services + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBE_SCHEDULER_OPTS="--address=10.64.33.81 \ + --algorithm-provider=DefaultProvider \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --port=10251 \ + --scheduler-name=default-scheduler \ + --v=2" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kubelet b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kubelet new file mode 100644 index 0000000..6b550b2 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/conf/kubelet @@ -0,0 +1,53 @@ +# KUBELET_VERSION=1.3.10 + +# --api-servers=[]: Comma separated list of Kubernetes API servers (ip:port). +# API_SERVERS=https://10.64.33.81:6443 + +# --hostname-override="": As identification instad of the actual hostname +# HOSTNAME_OVERRIDE=10.64.33.81 + +# --cert-dir="/var/run/kubernetes" +# CERT_DIR=/srv/kubernetes + +# CONFIG_PATH=/etc/kubernetes/manifests + +# --kubeconfig="/var/lib/kubelet/kubeconfig": Path to a kubeconfig file +# KUBECONFIG_PATH=/srv/kubernetes/kubeconfig + +# --master-service-namespace="default": The master services should be injected into pods + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# MASTER_ADDRESS=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBELET_OPTS="--address=10.64.33.81 \ + --allow-privileged=true \ + --api-servers=https://10.64.33.81:6443 \ + --cadvisor-port=4194 \ + --cert-dir=/srv/kubernetes \ + --cluster-dns=10.123.240.10 \ + --cluster-domain=cluster.local \ + --config=/etc/kubernetes/manifests \ + --configure-cbr0=false \ + --container-runtime=docker \ + --docker=unix:///var/run/docker.sock \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10248 \ + --hostname-override=10.64.33.81 \ + --kubeconfig=/srv/kubernetes/kubeconfig \ + --master-service-namespace=default \ + --max-open-files=1000000 \ + --max-pods=0 \ + --node-ip=10.64.33.81 \ + --non-masquerade-cidr=10.0.0.0/8 \ + --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ + --pods-per-core=0 \ + --port=10250 \ + --read-only-port=10255 \ + --root-dir=/var/lib/kubelet \ + --tls-cert-file=/srv/kubernetes/server.cert \ + --tls-private-key-file=/srv/kubernetes/server.key \ + --v=2" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-apiserver.service b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-apiserver.service new file mode 100644 index 0000000..2e59a9d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-apiserver.service @@ -0,0 +1,26 @@ +[Unit] +Description=Kubernetes API server +Documentation=https://github.com/kubernetes/kubernetes +After=etcd.service +Requires=etcd.service +# Wants= +# Conflicts=openshift-master.service + +[Service] +Type=notify +NotifyAccess=all +# Type=simple +User=root + +Environment=KUBERNETES_BUILD_VERSION=1.3.10 +EnvironmentFile=/etc/sysconfig/kube-apiserver +WorkingDirectory=/opt/kubernetes + +ExecStart=/opt/kubernetes/server/bin/kube-apiserver $KUBE_APISERVER_OPTS +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-apiserver ${KUBE_APISERVER_OPTS}" + +Restart=always +# Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-controller-manager.service b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-controller-manager.service new file mode 100644 index 0000000..f20feb7 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-controller-manager.service @@ -0,0 +1,26 @@ +[Unit] +Description=Kubernetes controller manager +Documentation=https://github.com/kubernetes/kubernetes +# After= +# Requires= +# Wants= +# Conflicts=openshift-master.service + +[Service] +# Type=notify +# NotifyAccess=none +Type=simple +User=root + +Environment=KUBERNETES_BUILD_VERSION=1.3.10 +EnvironmentFile=/etc/sysconfig/kube-controller-manager +WorkingDirectory=/opt/kubernetes + +ExecStart=/opt/kubernetes/server/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-controller-manager ${KUBE_CONTROLLER_MANAGER_OPTS}" + +# Restart=always +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-proxy.service b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-proxy.service new file mode 100644 index 0000000..fe3789f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-proxy.service @@ -0,0 +1,26 @@ +[Unit] +Description=Kubernetes proxy networking +Documentation=https://github.com/kubernetes/kubernetes +# After= +# Requires= +# Wants= +# Conflicts=openshift-node.service + +[Service] +# Type=notify +# NotifyAccess=none +Type=simple +User=root + +Environment=KUBERNETES_BUILD_VERSION=1.3.10 +EnvironmentFile=/etc/sysconfig/kube-proxy +WorkingDirectory=/opt/kubernetes + +ExecStart=/opt/kubernetes/server/bin/kube-proxy $KUBE_PROXY_OPTS +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-proxy ${KUBE_PROXY_OPTS}" + +# Restart=always +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-scheduler.service b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-scheduler.service new file mode 100644 index 0000000..b4c6d16 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kube-scheduler.service @@ -0,0 +1,26 @@ +[Unit] +Description=Kubernetes scheduler +Documentation=https://github.com/kubernetes/kubernetes +# After= +# Requires= +# Wants= +# Conflicts=openshift-master.service + +[Service] +# Type=notify +# NotifyAccess=main +Type=simple +User=root + +Environment=KUBE_BUILD_VERSION=1.3.10 +EnvironmentFile=/etc/sysconfig/kube-scheduler +WorkingDirectory=/opt/kubernetes + +ExecStart=/opt/kubernetes/server/bin/kube-scheduler $KUBE_SCHEDULER_OPTS +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-scheduler ${KUBE_SCHEDULER_OPTS}" + +# Restart=always +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kubelet.service b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kubelet.service new file mode 100644 index 0000000..07f767d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/centos/systemd/system/kubelet.service @@ -0,0 +1,30 @@ +[Unit] +Description=Kubernetes container mgmt daemon - kubelet +Documentation=https://github.com/kubernetes/kubernetes +After=docker.service +Requires=docker.service +# Wants= +Conflicts=openshift-node.service + +[Service] +#Type=notify +#NotifyAccess=all +Type=simple +User=root + +Environment=KUBERNETES_BUILD_VERSION=1.3.10 +EnvironmentFile=/etc/sysconfig/kubelet +WorkingDirectory=/opt/kubernetes + +ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests +ExecStartPre=/usr/bin/mkdir -p /var/log/containers + +ExecStart=/opt/kubernetes/server/bin/kubelet $KUBELET_OPTS +#ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kubelet ${KUBELET_OPTS}" + +Restart=on-failure +RestartSec=10 +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/conf/kube-proxy b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/conf/kube-proxy new file mode 100755 index 0000000..355adf3 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/conf/kube-proxy @@ -0,0 +1,28 @@ +# KUBE_PROXY_VERSION=1.3.10 + +# BIND_ADDRESS=10.64.33.90 +# HOSTNAME_OVERRIDE=10.64.33.90 + +# --iptables-masquerade-bit=14: If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within [0, 31] + +# KUBECONFIG_PATH=/etc/kubernetes/cacerts/kubeconfig +# MASTER_ADDRESS=https://10.64.33.81:6443 + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBE_PROXY_OPTS=" \ + --bind-address=10.64.33.90 \ + --cluster-cidr=10.120.0.0/14 \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10249 \ + --hostname-override=10.64.33.90 \ + --iptables-masquerade-bit=14 \ + --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --v=2 \ +" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/conf/kubelet b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/conf/kubelet new file mode 100755 index 0000000..debaae6 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/conf/kubelet @@ -0,0 +1,69 @@ +# KUBELET_VERSION=1.3.10 + +# --api-servers=[]: Comma separated list of Kubernetes API servers (ip:port). +# API_SERVERS=https://10.64.33.81:6443 + +# --cert-dir="/var/run/kubernetes" + +# --config=: The path including auto-start POD manifests +# CONFIG_PATH=/etc/kubernetes/manifests + +# --hostname-override="": As identification instad of the actual hostname +# HOSTNAME_OVERRIDE=10.64.33.90 + +# --kubeconfig="/var/lib/kubelet/kubeconfig": Path to a kubeconfig file +# KUBECONFIG_PATH=/etc/kubernetes/cacerts/kubeconfig + +# --master-service-namespace="default": The master should be injected into pods + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.123.240.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +# TLS_SERVER_CERT=/etc/kubernetes/cacerts/kubelet.crt +# TLS_SERVER_KEY=/etc/kubernetes/cacerts/kubelet.key + +KUBELET_OPTS=" \ + --address=10.64.33.90 \ + --allow-privileged=true \ + --api-servers=https://10.64.33.81:6443 \ + --cluster-dns=10.123.240.10 \ + --cluster-domain=cluster.local \ + --config=/etc/kubernetes/manifests \ + --hostname-override=10.64.33.90 \ + --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ + --node-ip=10.64.33.90 \ + --v=2 \ +" + +# KUBELET_OPTS="--address=10.64.33.90 \ +# --allow-privileged=true \ +# --api-servers=https://10.64.33.81:6443 \ +# --cadvisor-port=4194 \ +# --cert-dir=/var/run/kubernetes \ +# --cluster-dns=10.123.240.10 \ +# --cluster-domain=cluster.local \ +# --config=/etc/kubernetes/manifests \ +# --configure-cbr0=false \ +# --container-runtime=docker \ +# --docker=unix:///var/run/docker.sock \ +# --healthz-bind-address=127.0.0.1 \ +# --healthz-port=10248 \ +# --hostname-override=10.64.33.90 \ +# --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ +# --master-service-namespace=default \ +# --max-open-files=1000000 \ +# --max-pods=110 \ +# --node-ip=10.64.33.90 \ +# --non-masquerade-cidr=10.0.0.0/8 \ +# --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ +# --pods-per-core=0 \ +# --port=10250 \ +# --read-only-port=10255 \ +# --root-dir=/var/lib/kubelet \ +# --tls-cert-file=/etc/kubernetes/cacerts/kubelet.crt \ +# --tls-private-key-file=/etc/kubernetes/cacerts/kubelet.key \ +# --v=2" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/service/kube-proxy.service b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/service/kube-proxy.service new file mode 100755 index 0000000..5507793 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/service/kube-proxy.service @@ -0,0 +1,27 @@ +[Unit] +Description=Kubernetes proxy networking +Documentation=https://github.com/kubernetes/kubernetes +# After= +# Requires= +# Wants= +# Conflicts=openshift-node.service + +[Service] +# Type=notify +# NotifyAccess=none +Type=simple +# User=root + +EnvironmentFile=/etc/sysconfig/kube-proxy + +# WorkingDirectory=/opt/kubernetes +Environment=KUBERNETES_RELEASE_VERSION=1.3.1 + +ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_RELEASE_VERSION}/fedora23/bin/kube-proxy ${KUBE_PROXY_OPTS}" +# ExecStart=/opt/kubernetes/server/bin/kube-proxy $KUBE_PROXY_OPTS + +# Restart=always +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/service/kubelet.service b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/service/kubelet.service new file mode 100755 index 0000000..0bba81b --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/fedora23/systemd/service/kubelet.service @@ -0,0 +1,32 @@ +[Unit] +Description=Kubernetes container mgmt daemon - kubelet +Documentation=https://github.com/kubernetes/kubernetes +After=docker.service +Requires=docker.service +# Wants= +Conflicts=openshift-node.service + +[Service] +Type=simple +# Type=notify +# NotifyAccess=all +# User=root + +EnvironmentFile=/etc/sysconfig/kubelet + +# WorkingDirectory=/var/lib/kubelet +Environment=KUBERNETES_RELEASE_VERSION=1.3.1 + +# ExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet +ExecStartPre=/usr/bin/mkdir -p /var/log/containers + +ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_RELEASE_VERSION}/fedora23/bin/kubelet ${KUBELET_OPTS}" +# ExecStart=/opt/kubernetes/server/bin/kubelet $KUBELET_OPTS + +# Restart=always +Restart=on-failure +RestartSec=10 +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dashboard-rc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dashboard-rc.yaml new file mode 100755 index 0000000..5f5a32d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dashboard-rc.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kubernetes-dashboard-v1.4.0 + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + version: v1.4.0 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + version: v1.4.0 + kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubernetes-dashboard + image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.4.0 + resources: + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + ports: + - containerPort: 9090 + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dashboard-svc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dashboard-svc.yaml new file mode 100755 index 0000000..74b072f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dashboard-svc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 80 + targetPort: 9090 diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dns-rc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dns-rc.yaml new file mode 100755 index 0000000..8d863cd --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dns-rc.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v19 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v19 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-dns + version: v19 + template: + metadata: + labels: + k8s-app: kube-dns + version: v19 + kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubedns + image: gcr.io/google_containers/kubedns-amd64:1.7 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + args: + - --domain=cluster.local. + - --dns-port=10053 + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - name: dnsmasq + image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3 + args: + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + - --log-facility=- + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz-amd64:1.1 + resources: + limits: + memory: 50Mi + requests: + cpu: 10m + memory: 50Mi + args: + - -cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc.cluster.local 127.0.0.1:10053 >/dev/null + - -port=8080 + - -quiet + ports: + - containerPort: 8080 + protocol: TCP + dnsPolicy: Default diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dns-svc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dns-svc.yaml new file mode 100755 index 0000000..7168802 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.10/manifests/addons/kube-dns-svc.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.123.240.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dashboard-rc.json b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dashboard-rc.json new file mode 100755 index 0000000..e1f839b --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dashboard-rc.json @@ -0,0 +1,59 @@ +{ + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "labels": { + "k8s-app": "kubernetes-dashboard", + "kubernetes.io/cluster-service": "true", + "version": "v1.1.0" + }, + "name": "kubernetes-dashboard-v1.1.0", + "namespace": "kube-system" + }, + "spec": { + "replicas": 1, + "selector": { + "k8s-app": "kubernetes-dashboard" + }, + "template": { + "metadata": { + "labels": { + "k8s-app": "kubernetes-dashboard", + "kubernetes.io/cluster-service": "true", + "version": "v1.1.0" + } + }, + "spec": { + "containers": [ + { + "image": "gcr.io/google_containers/kubernetes-dashboard-amd64:v1.1.0", + "livenessProbe": { + "httpGet": { + "path": "/", + "port": 9090 + }, + "initialDelaySeconds": 30, + "timeoutSeconds": 30 + }, + "name": "kubernetes-dashboard", + "ports": [ + { + "containerPort": 9090 + } + ], + "resources": { + "limits": { + "cpu": "100m", + "memory": "50Mi" + }, + "requests": { + "cpu": "100m", + "memory": "50Mi" + } + } + } + ] + } + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dashboard-svc.json b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dashboard-svc.json new file mode 100755 index 0000000..190c62f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dashboard-svc.json @@ -0,0 +1,23 @@ +{ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "k8s-app": "kubernetes-dashboard", + "kubernetes.io/cluster-service": "true" + }, + "name": "kubernetes-dashboard", + "namespace": "kube-system" + }, + "spec": { + "ports": [ + { + "port": 80, + "targetPort": 9090 + } + ], + "selector": { + "k8s-app": "kubernetes-dashboard" + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dns-rc.json b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dns-rc.json new file mode 100755 index 0000000..4678000 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dns-rc.json @@ -0,0 +1,130 @@ +{ + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "version": "v15" + }, + "name": "kube-dns-v15", + "namespace": "kube-system" + }, + "spec": { + "replicas": 1, + "selector": { + "k8s-app": "kube-dns", + "version": "v15" + }, + "template": { + "metadata": { + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "version": "v15" + } + }, + "spec": { + "containers": [ + { + "args": [ + "--domain=cluster.local.", + "--dns-port=10053" + ], + "image": "gcr.io/google_containers/kubedns-amd64:1.3", + "livenessProbe": { + "failureThreshold": 5, + "httpGet": { + "path": "/healthz", + "port": 8080, + "scheme": "HTTP" + }, + "initialDelaySeconds": 60, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "name": "kubedns", + "ports": [ + { + "containerPort": 10053, + "name": "dns-local", + "protocol": "UDP" + }, + { + "containerPort": 10053, + "name": "dns-tcp-local", + "protocol": "TCP" + } + ], + "readinessProbe": { + "httpGet": { + "path": "/readiness", + "port": 8081, + "scheme": "HTTP" + }, + "initialDelaySeconds": 30, + "timeoutSeconds": 5 + }, + "resources": { + "limits": { + "cpu": "100m", + "memory": "200Mi" + }, + "requests": { + "cpu": "100m", + "memory": "50Mi" + } + } + }, + { + "args": [ + "--cache-size=1000", + "--no-resolv", + "--server=127.0.0.1#10053" + ], + "image": "gcr.io/google_containers/kube-dnsmasq-amd64:1.3", + "name": "dnsmasq", + "ports": [ + { + "containerPort": 53, + "name": "dns", + "protocol": "UDP" + }, + { + "containerPort": 53, + "name": "dns-tcp", + "protocol": "TCP" + } + ] + }, + { + "args": [ + "-cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null", + "-port=8080", + "-quiet" + ], + "image": "gcr.io/google_containers/exechealthz-amd64:1.0", + "name": "healthz", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ], + "resources": { + "limits": { + "cpu": "10m", + "memory": "20Mi" + }, + "requests": { + "cpu": "10m", + "memory": "20Mi" + } + } + } + ], + "dnsPolicy": "Default" + } + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dns-svc.json b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dns-svc.json new file mode 100755 index 0000000..3fb9530 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.4/manifests/addons/kube-dns-svc.json @@ -0,0 +1,31 @@ +{ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "kubernetes.io/name": "KubeDNS" + }, + "name": "kube-dns", + "namespace": "kube-system" + }, + "spec": { + "clusterIP": "10.123.240.10", + "ports": [ + { + "name": "dns", + "port": 53, + "protocol": "UDP" + }, + { + "name": "dns-tcp", + "port": 53, + "protocol": "TCP" + } + ], + "selector": { + "k8s-app": "kube-dns" + } + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dashboard/dashboard-controller.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dashboard/dashboard-controller.yaml new file mode 100755 index 0000000..28d8230 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dashboard/dashboard-controller.yaml @@ -0,0 +1,43 @@ +# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml +apiVersion: v1 +kind: ReplicationController +metadata: + name: kubernetes-dashboard-v1.4.0 + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + version: v1.4.0 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + version: v1.4.0 + kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubernetes-dashboard + image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.4.0 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + ports: + - containerPort: 9090 + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dashboard/dashboard-service.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dashboard/dashboard-service.yaml new file mode 100755 index 0000000..2c1c02f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dashboard/dashboard-service.yaml @@ -0,0 +1,15 @@ +# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 80 + targetPort: 9090 diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-rc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-rc.yaml new file mode 100644 index 0000000..f98c599 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-rc.yaml @@ -0,0 +1,129 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* + +# Warning: This is a file generated from the base underscore template file: skydns-rc.yaml.base + +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v20 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v20 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-dns + version: v20 + template: + metadata: + labels: + k8s-app: kube-dns + version: v20 + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubedns + image: gcr.io/google_containers/kubedns-amd64:1.8 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthz-kubedns + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + # command = "/kube-dns" + - --domain=cluster.local. + - --dns-port=10053 + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - name: dnsmasq + image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4 + livenessProbe: + httpGet: + path: /healthz-dnsmasq + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + - --log-facility=- + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz-amd64:1.2 + resources: + limits: + memory: 50Mi + requests: + cpu: 10m + # Note that this container shouldn't really need 50Mi of memory. The + # limits are set higher than expected pending investigation on #29688. + # The extra memory was stolen from the kubedns container to keep the + # net memory requested by the pod constant. + memory: 50Mi + args: + - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null + - --url=/healthz-dnsmasq + - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1:10053 >/dev/null + - --url=/healthz-kubedns + - --port=8080 + - --quiet + ports: + - containerPort: 8080 + protocol: TCP + dnsPolicy: Default # Don't use cluster DNS. diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-svc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-svc.yaml new file mode 100644 index 0000000..10e8845 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-svc.yaml @@ -0,0 +1,38 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* + +# Warning: This is a file generated from the base underscore template file: skydns-svc.yaml.base + +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.123.240.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-apiserver b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-apiserver new file mode 100644 index 0000000..8788e28 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-apiserver @@ -0,0 +1,123 @@ +# KUBE_APISERVER_VERSION=1.3.7 + +# --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota + +# --advertise-address=: IP address to advertise the apiserver to members of the cluster, must be reachable by the cluster +# ADVERTISE_APISERVER=10.64.33.81 + +# --cert-dir="/var/run/kubernetes" + +# --enable-swagger-ui[=false]: Enables swagger ui at /swagger-ui +# ENABLE_SWAGGER_UI=false + +# --etcd-servers=: comma separated etcd servers to connect (http://ip:port) +# ETCD_SERVERS=http://10.64.33.81:2379 + +# --master-service-namespace="default": The master should be injected into pods + +# --runtime-config=: A set of key=value pairs such as apis/=true/false, also apis// +# RUNTIME_CONFIG=api/all=true + +# --secure-port=6443: The port on which to serve HTTPS +# SECURE_PORT=6443 + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +# --service-node-port-range=: default '30000-32767' + +# +# OIDC plugin ready +# + +KUBE_APISERVER_OPTS=" \ + --admission-control=SecurityContextDeny,ServiceAccount \ + --advertise-address=10.64.33.81 \ + --allow-privileged=true \ + --apiserver-count=1 \ + --bind-address=0.0.0.0 \ + --cert-dir=/var/run/kubernetes \ + --client-ca-file=/etc/kubernetes/cacerts/ca.crt \ + --enable-garbage-collector=false \ + --enable-swagger-ui=false \ + --etcd-servers=http://10.64.33.81:2379 \ + --insecure-bind-address=10.64.33.81 \ + --insecure-port=8080 \ + --kubelet-https=true \ + --master-service-namespace=default \ + --oidc-issuer-url=https://www.10.64.33.90.xip.io:5556 \ + --oidc-client-id=example-app \ + --oidc-ca-file=/etc/coreos/dex/ca.crt \ + --oidc-username-claim=email \ + --oidc-groups-claim=groups \ + --runtime-config=api/all=true \ + --secure-port=6443 \ + --service-account-key-file=/etc/kubernetes/cacerts/server.key \ + --service-cluster-ip-range=10.123.240.0/20 \ + --tls-cert-file=/etc/kubernetes/cacerts/server.cert \ + --tls-private-key-file=/etc/kubernetes/cacerts/server.key \ + --v=2 \ +" + +# +# X509 only +# + +# KUBE_APISERVER_OPTS=" \ +# --admission-control=SecurityContextDeny,ServiceAccount \ +# --advertise-address=10.64.33.81 \ +# --allow-privileged=true \ +# --apiserver-count=1 \ +# --bind-address=0.0.0.0 \ +# --cert-dir=/var/run/kubernetes \ +# --client-ca-file=/etc/kubernetes/cacerts/ca.crt \ +# --enable-garbage-collector=false \ +# --enable-swagger-ui=false \ +# --etcd-servers=http://10.64.33.81:2379 \ +# --insecure-bind-address=10.64.33.81 \ +# --insecure-port=8080 \ +# --kubelet-https=true \ +# --master-service-namespace=default \ +# --runtime-config=api/all=true \ +# --secure-port=6443 \ +# --service-account-key-file=/etc/kubernetes/cacerts/server.key \ +# --service-cluster-ip-range=10.123.240.0/20 \ +# --tls-cert-file=/etc/kubernetes/cacerts/server.cert \ +# --tls-private-key-file=/etc/kubernetes/cacerts/server.key \ +# --v=2 \ +# " + +# +# kubelet certification +# + +# KUBE_APISERVER_OPTS=" \ +# --admission-control=SecurityContextDeny,ServiceAccount \ +# --advertise-address=10.64.33.81 \ +# --allow-privileged=true \ +# --apiserver-count=1 \ +# --bind-address=0.0.0.0 \ +# --cert-dir=/var/run/kubernetes \ +# --client-ca-file=/etc/kubernetes/cacerts/ca.crt \ +# --enable-garbage-collector=false \ +# --enable-swagger-ui=false \ +# --etcd-servers=http://10.64.33.81:2379 \ +# --insecure-bind-address=10.64.33.81 \ +# --insecure-port=8080 \ +# --kubelet-certificate-authority=/etc/kubernetes/cacerts/ca.crt \ +# --kubelet-client-certificate=/etc/kubernetes/cacerts/kubelet.cert \ +# --kubelet-client-key=/etc/kubernetes/cacerts/kubelet.key \ +# --kubelet-https=true \ +# --master-service-namespace=default \ +# --runtime-config=api/all=true \ +# --secure-port=6443 \ +# --service-account-key-file=/etc/kubernetes/cacerts/server.key \ +# --service-cluster-ip-range=10.123.240.0/20 \ +# --tls-cert-file=/etc/kubernetes/cacerts/server.cert \ +# --tls-private-key-file=/etc/kubernetes/cacerts/server.key \ +# --v=2 \ +# " diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-controller-manager b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-controller-manager new file mode 100644 index 0000000..21d37fe --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-controller-manager @@ -0,0 +1,30 @@ +# KUBE_CONTROLLER_MANAGER_VERSION=1.3.10 + +# ADDRESS_LISTEN=10.64.33.81 + +# --cloud-config="": The path to the cloud provider configuration file + +# --cloud-provider="": The provider for cloud services + +# MASTER_ADDRESS=https://10.64.33.81:6443 + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBE_CONTROLLER_MANAGER_OPTS=" \ + --address=10.64.33.81 \ + --cluster-cidr=10.120.0.0/14 \ + --cluster-name=kubernetes \ + --enable-garbage-collector=false \ + --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --port=10252 \ + --root-ca-file=/etc/kubernetes/cacerts/ca.crt \ + --service-account-private-key-file=/etc/kubernetes/cacerts/server.key \ + --service-cluster-ip-range=10.123.240.0/20 \ + --v=2 \ +" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-proxy b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-proxy new file mode 100644 index 0000000..070ee29 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-proxy @@ -0,0 +1,28 @@ +# KUBE_PROXY_VERSION=1.3.7 + +# BIND_ADDRESS=10.64.33.81 +# HOSTNAME_OVERRIDE=10.64.33.81 + +# --iptables-masquerade-bit=14: If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within [0, 31] + +# KUBECONFIG_PATH=/srv/kubernetes/kubeconfig +# MASTER_ADDRESS=https://10.64.33.81:6443 + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBE_PROXY_OPTS=" \ + --bind-address=10.64.33.81 \ + --cluster-cidr=10.120.0.0/14 \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10249 \ + --hostname-override=10.64.33.81 \ + --iptables-masquerade-bit=14 \ + --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --v=2 \ +" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-scheduler b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-scheduler new file mode 100644 index 0000000..ca1521e --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kube-scheduler @@ -0,0 +1,24 @@ +# KUBE_SCHEDULER_VERSION=1.3.7 + +# ADDRESS_LISTEN=10.64.33.81 +# MASTER_ADDRESS=https://10.64.33.81:6443 + +# --cloud-config="": The path to the cloud provider configuration file +# --cloud-provider="": The provider for cloud services + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +KUBE_SCHEDULER_OPTS=" \ + --address=10.64.33.81 \ + --algorithm-provider=DefaultProvider \ + --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ + --master=https://10.64.33.81:6443 \ + --port=10251 \ + --scheduler-name=default-scheduler \ + --v=2 \ +" diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kubelet b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kubelet new file mode 100644 index 0000000..08fa763 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/conf/kubelet @@ -0,0 +1,87 @@ +# KUBELET_VERSION=1.3.7 + +# --api-servers=[]: Comma separated list of Kubernetes API servers (ip:port). +# API_SERVERS=https://10.64.33.81:6443 + +# --cert-dir="/var/run/kubernetes" + +# --config=: The path including auto-start POD manifests +# CONFIG_PATH=/etc/kubernetes/manifests + +# --hostname-override="": As identification instad of the actual hostname +# HOSTNAME_OVERRIDE=10.64.33.81 + +# --kubeconfig="/var/lib/kubelet/kubeconfig": Path to a kubeconfig file +# KUBECONFIG_PATH=/etc/kubernetes/cacerts/kubeconfig + +# --master-service-namespace="default": The master should be injected into pods + +# --service-cluster-ip-range=: A CIDR notation IP range +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +# CLUSTER_CIDR=10.120.0.0/14 +# SERVICE_CIDR=10.123.240.0/20 +# CLUSTER_MASTER=10.123.240.1 +# CLUSTER_DNS=10.123.240.10 + +# TLS_SERVER_CERT=/etc/kubernetes/cacerts/kubelet.crt +# TLS_SERVER_KEY=/etc/kubernetes/cacerts/kubelet.key + +KUBELET_OPTS=" \ + --address=10.64.33.81 \ + --allow-privileged=true \ + --api-servers=https://10.64.33.81:6443 \ + --cadvisor-port=4194 \ + --cert-dir=/var/run/kubernetes \ + --cluster-dns=10.123.240.10 \ + --cluster-domain=cluster.local \ + --config=/etc/kubernetes/manifests \ + --configure-cbr0=false \ + --container-runtime=docker \ + --docker=unix:///var/run/docker.sock \ + --healthz-bind-address=127.0.0.1 \ + --healthz-port=10248 \ + --hostname-override=10.64.33.81 \ + --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ + --master-service-namespace=default \ + --max-open-files=1000000 \ + --max-pods=110 \ + --node-ip=10.64.33.81 \ + --non-masquerade-cidr=10.0.0.0/8 \ + --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ + --pods-per-core=0 \ + --port=10250 \ + --read-only-port=10255 \ + --root-dir=/var/lib/kubelet \ + --v=2 \ +" + +# KUBELET_OPTS=" \ +# --address=10.64.33.81 \ +# --allow-privileged=true \ +# --api-servers=https://10.64.33.81:6443 \ +# --cadvisor-port=4194 \ +# --cert-dir=/var/run/kubernetes \ +# --cluster-dns=10.123.240.10 \ +# --cluster-domain=cluster.local \ +# --config=/etc/kubernetes/manifests \ +# --configure-cbr0=false \ +# --container-runtime=docker \ +# --docker=unix:///var/run/docker.sock \ +# --healthz-bind-address=127.0.0.1 \ +# --healthz-port=10248 \ +# --hostname-override=10.64.33.81 \ +# --kubeconfig=/etc/kubernetes/cacerts/kubeconfig \ +# --master-service-namespace=default \ +# --max-open-files=1000000 \ +# --max-pods=110 \ +# --node-ip=10.64.33.81 \ +# --non-masquerade-cidr=10.0.0.0/8 \ +# --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 \ +# --pods-per-core=0 \ +# --port=10250 \ +# --read-only-port=10255 \ +# --root-dir=/var/lib/kubelet \ +# --tls-cert-file=/etc/kubernetes/cacerts/kubelet.crt \ +# --tls-private-key-file=/etc/kubernetes/cacerts/kubelet.key \ +# --v=2 \ +# " diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-apiserver.service b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-apiserver.service new file mode 100644 index 0000000..1cdc88f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-apiserver.service @@ -0,0 +1,26 @@ +[Unit] +Description=Kubernetes API server +Documentation=https://github.com/kubernetes/kubernetes +After=etcd.service +Requires=etcd.service +# Wants= +# Conflicts=openshift-master.service + +[Service] +Type=notify +NotifyAccess=all +# Type=simple +User=root + +EnvironmentFile=/etc/sysconfig/kube-apiserver +WorkingDirectory=/opt/kubernetes +Environment=KUBERNETES_BUILD_VERSION=1.3.7 + +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-apiserver ${KUBE_APISERVER_OPTS}" +ExecStart=/opt/kubernetes/server/bin/kube-apiserver $KUBE_APISERVER_OPTS + +Restart=always +# Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-controller-manager.service b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-controller-manager.service new file mode 100644 index 0000000..ef3e306 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-controller-manager.service @@ -0,0 +1,26 @@ +[Unit] +Description=Kubernetes controller manager +Documentation=https://github.com/kubernetes/kubernetes +# After= +# Requires= +# Wants= +# Conflicts=openshift-master.service + +[Service] +# Type=notify +# NotifyAccess=none +Type=simple +User=root + +EnvironmentFile=/etc/sysconfig/kube-controller-manager +# WorkingDirectory=/opt/kubernetes +Environment=KUBERNETES_BUILD_VERSION=1.3.10 + +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-controller-manager ${KUBE_CONTROLLER_MANAGER_OPTS}" +ExecStart=/opt/kubernetes/server/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS + +# Restart=always +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-proxy.service b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-proxy.service new file mode 100644 index 0000000..9c138d2 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-proxy.service @@ -0,0 +1,26 @@ +[Unit] +Description=Kubernetes proxy networking +Documentation=https://github.com/kubernetes/kubernetes +# After= +# Requires= +# Wants= +# Conflicts=openshift-node.service + +[Service] +# Type=notify +# NotifyAccess=none +Type=simple +# User=root + +EnvironmentFile=/etc/sysconfig/kube-proxy +# WorkingDirectory=/opt/kubernetes +Environment=KUBERNETES_BUILD_VERSION=1.3.7 + +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-proxy ${KUBE_PROXY_OPTS}" +ExecStart=/opt/kubernetes/server/bin/kube-proxy $KUBE_PROXY_OPTS + +# Restart=always +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-scheduler.service b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-scheduler.service new file mode 100644 index 0000000..96b76fa --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kube-scheduler.service @@ -0,0 +1,27 @@ +[Unit] +Description=Kubernetes scheduler +Documentation=https://github.com/kubernetes/kubernetes +# After= +# Requires= +# Wants= +# Conflicts=openshift-master.service + +[Service] +# Type=notify +# NotifyAccess=main +Type=simple +User=root + +EnvironmentFile=/etc/sysconfig/kube-scheduler +# WorkingDirectory=/opt/kubernetes +Environment=KUBE_BUILD_VERSION=1.3.7 + +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kube-scheduler ${KUBE_SCHEDULER_OPTS}" +ExecStart=/opt/kubernetes/server/bin/kube-scheduler $KUBE_SCHEDULER_OPTS + +# Restart=always +Restart=on-failure + +[Install] +WantedBy=multi-user.target + diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kubelet.service b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kubelet.service new file mode 100644 index 0000000..f0f00c5 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/centos/systemd/system/kubelet.service @@ -0,0 +1,31 @@ +[Unit] +Description=Kubernetes container mgmt daemon - kubelet +Documentation=https://github.com/kubernetes/kubernetes +After=docker.service +Requires=docker.service +# Wants= +Conflicts=openshift-node.service + +[Service] +# Type=notify +# NotifyAccess=all +Type=simple +# User=root + +EnvironmentFile=/etc/sysconfig/kubelet +# WorkingDirectory=/var/lib/kubelet +Environment=KUBERNETES_BUILD_VERSION=1.3.7 + +# ExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet +ExecStartPre=/usr/bin/mkdir -p /var/log/containers + +# ExecStart=/bin/bash -c "/opt/kubernetes/${KUBERNETES_BUILD_VERSION}/centos/bin/kubelet ${KUBELET_OPTS}" +ExecStart=/opt/kubernetes/server/bin/kubelet $KUBELET_OPTS + +# Restart=always +Restart=on-failure +RestartSec=10 +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dashboard/dashboard-controller.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dashboard/dashboard-controller.yaml new file mode 100755 index 0000000..28d8230 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dashboard/dashboard-controller.yaml @@ -0,0 +1,43 @@ +# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-controller.yaml +apiVersion: v1 +kind: ReplicationController +metadata: + name: kubernetes-dashboard-v1.4.0 + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + version: v1.4.0 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + version: v1.4.0 + kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubernetes-dashboard + image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.4.0 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + ports: + - containerPort: 9090 + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dashboard/dashboard-service.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dashboard/dashboard-service.yaml new file mode 100755 index 0000000..2c1c02f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dashboard/dashboard-service.yaml @@ -0,0 +1,15 @@ +# This file should be kept in sync with cluster/gce/coreos/kube-manifests/addons/dashboard/dashboard-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 80 + targetPort: 9090 diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dns/skydns-rc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dns/skydns-rc.yaml new file mode 100644 index 0000000..b57b59a --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dns/skydns-rc.yaml @@ -0,0 +1,122 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file should be kept in sync with cluster/images/hyperkube/dns-rc.yaml + +# TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* + +# Warning: This is a file generated from the base underscore template file: skydns-rc.yaml.base + +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v19 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v19 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-dns + version: v19 + template: + metadata: + labels: + k8s-app: kube-dns + version: v19 + kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubedns + image: gcr.io/google_containers/kubedns-amd64:1.7 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 30 + timeoutSeconds: 5 + args: + # command = "/kube-dns" + - --domain=cluster.local. + - --dns-port=10053 + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - name: dnsmasq + image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3 + args: + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz-amd64:1.1 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 10m + memory: 50Mi + requests: + cpu: 10m + # Note that this container shouldn't really need 50Mi of memory. The + # limits are set higher than expected pending investigation on #29688. + # The extra memory was stolen from the kubedns container to keep the + # net memory requested by the pod constant. + memory: 50Mi + args: + - -cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc.cluster.local 127.0.0.1:10053 >/dev/null + - -port=8080 + - -quiet + ports: + - containerPort: 8080 + protocol: TCP + dnsPolicy: Default # Don't use cluster DNS. diff --git a/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dns/skydns-svc.yaml b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dns/skydns-svc.yaml new file mode 100644 index 0000000..24c567f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.3.7/manifests/addons/dns/skydns-svc.yaml @@ -0,0 +1,40 @@ +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file should be kept in sync with cluster/images/hyperkube/dns-svc.yaml + +# TODO - At some point, we need to rename all skydns-*.yaml.* files to kubedns-*.yaml.* + +# Warning: This is a file generated from the base underscore template file: skydns-svc.yaml.base + +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.123.240.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dashboard-rc.yaml b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dashboard-rc.yaml new file mode 100755 index 0000000..de6db14 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dashboard-rc.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kubernetes-dashboard-v1.4.1 + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + version: v1.4.1 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + version: v1.4.1 + kubernetes.io/cluster-service: "true" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubernetes-dashboard + image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.4.1 + resources: + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + ports: + - containerPort: 9090 + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 diff --git a/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dashboard-svc.yaml b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dashboard-svc.yaml new file mode 100755 index 0000000..74b072f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dashboard-svc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 80 + targetPort: 9090 diff --git a/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dns-rc.yaml b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dns-rc.yaml new file mode 100755 index 0000000..0b28e88 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dns-rc.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v20 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v20 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-dns + version: v20 + template: + metadata: + labels: + k8s-app: kube-dns + version: v20 + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubedns + image: gcr.io/google_containers/kubedns-amd64:1.8 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthz-kubedns + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + - --domain=cluster.local. + - --dns-port=10053 + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - name: dnsmasq + image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4 + livenessProbe: + httpGet: + path: /healthz-dnsmasq + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + - --log-facility=- + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz-amd64:1.2 + resources: + limits: + memory: 50Mi + requests: + cpu: 10m + memory: 50Mi + args: + - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null + - --url=/healthz-dnsmasq + - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1:10053 >/dev/null + - --url=/healthz-kubedns + - --port=8080 + - --quiet + ports: + - containerPort: 8080 + protocol: TCP + dnsPolicy: Default diff --git a/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dns-svc.yaml b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dns-svc.yaml new file mode 100755 index 0000000..7168802 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/1.4.3/manifests/addons/kube-dns-svc.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.123.240.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/addons.md b/hack/kubernetes-clearwater/kubernetes/addons.md new file mode 100755 index 0000000..4eb1cac --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/addons.md @@ -0,0 +1,36 @@ +Instruction +------------ + +For `kubernetes` 1.3.6 + + tangf@DESKTOP-H68OQDV ~ + $ cd /cygdrive/g/go/src/k8s.io/kubernetes/ + + tangf@DESKTOP-H68OQDV /cygdrive/g/go/src/k8s.io/kubernetes + $ git branch --all + * master + remotes/origin/1.3.6 + remotes/origin/1.3.7 + remotes/origin/HEAD -> origin/master + + tangf@DESKTOP-H68OQDV /cygdrive/g/go/src/k8s.io/kubernetes + $ git checkout 1.3.6 + Checking out files: 100% (8487/8487), done. + Branch 1.3.6 set up to track remote branch 1.3.6 from origin. + Switched to a new branch '1.3.6' + +DNS + + tangf@DESKTOP-H68OQDV /cygdrive/g/go/src/k8s.io/kubernetes + $ mkdir -p /cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns + + tangf@DESKTOP-H68OQDV /cygdrive/g/go/src/k8s.io/kubernetes + $ sed "s/\$DNS_REPLICAS/1/g;s/\$DNS_DOMAIN/cluster.local/g" cluster/addons/dns/skydns-rc.yaml.sed >/cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-rc.yaml + + tangf@DESKTOP-H68OQDV /cygdrive/g/go/src/k8s.io/kubernetes + $ sed "s/\$DNS_SERVER_IP/10.123.240.10/g" cluster/addons/dns/skydns-svc.yaml.sed >/cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dns/skydns-svc.yaml + +Dashboard + + tangf@DESKTOP-H68OQDV /cygdrive/g/go/src/k8s.io/kubernetes + $ cp cluster/addons/dashboard/dashboard-*.yaml /cygdrive/g/work/src/github.com/Metaswitch/clearwater-docker/hack/kubernetes-clearwater/kubernetes/1.3.6/manifests/addons/dashboard/ diff --git a/hack/kubernetes-clearwater/kubernetes/admin-client.md b/hack/kubernetes-clearwater/kubernetes/admin-client.md new file mode 100755 index 0000000..f065502 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/admin-client.md @@ -0,0 +1,68 @@ +### Tables of content + +* KUBECONFIG of kubectl + +* Private registry of docker + +Config for `kubectl` + + [vagrant@localhost ~]$ ip a show eth1 + 3: eth1: mtu 1500 qdisc fq_codel state UP group default qlen 1000 + link/ether 08:00:27:74:ca:21 brd ff:ff:ff:ff:ff:ff + inet 10.64.33.101/24 brd 10.64.33.255 scope global eth1 + valid_lft forever preferred_lft forever + inet6 fe80::a00:27ff:fe74:ca21/64 scope link + valid_lft forever preferred_lft forever + + [vagrant@localhost ~]$ mkdir -p .pki/kubernetes/master81 + + [vagrant@master81 ~]$ scp .kube/config vagrant@10.64.33.101:/home/vagrant/.pki/kubernetes/master81/kubeconfig + vagrant@10.64.33.101's password: + config 100% 10KB 9.9KB/s 00:00 + + [vagrant@localhost ~]$ KUBECONFIG='.pki/kubernetes/master81/kubeconfig' kubectl get nodes + NAME STATUS AGE + 10.64.33.81 Ready 1d + +Private `registry` for `docker` + + [vagrant@localhost ~]$ KUBECONFIG='.pki/kubernetes/master81/kubeconfig' kubectl --namespace=stackdocker get pods + NAME READY STATUS RESTARTS AGE + registry-2499285598-9vdxa 1/1 Running 1 15h + + [vagrant@localhost ~]$ scp vagrant@10.64.33.81:/home/vagrant/kubernetes/registry/2.4.1/tls.* . + vagrant@10.64.33.81's password: + tls.crt 100% 1285 1.3KB/s 00:00 + tls.key 100% 1679 1.6KB/s 00:00 + + [vagrant@localhost ~]$ sudo mkdir -p /etc/docker/certs.d/10.64.33.81:5000 + + [vagrant@localhost ~]$ sudo mv tls.* /etc/docker/certs.d/10.64.33.81\:5000/ + + [vagrant@localhost ~]$ sudo vi /etc/sysconfig/docker + OPTIONS='--insecure-registry=172.30.0.0/16 --insecure-registry=ci.dev.openshift.redhat.com:5000 --selinux-enabled --log-driver=journald --insecure-registry=10.128.0.0/14 --bridge=lbr0 --fixed-cidr=10.128.2.0/23 --insecure-registry=10.64.33.81:5000' + + [vagrant@localhost ~]$ sudo systemctl restart docker.service + + [vagrant@localhost ~]$ docker login -u admin -p admin123 10.64.33.81:5000 + Email: + WARNING: login credentials saved in /home/vagrant/.docker/config.json + Login Succeeded + + [vagrant@master81 ~]$ docker tag gcr.io/google_containers/pause-amd64:3.0 10.64.33.81:5000/google_containers_pause-amd64:3.0 + + [vagrant@master81 ~]$ docker push 10.64.33.81:5000/google_containers_pause-amd64:3.0 + The push refers to a repository [10.64.33.81:5000/google_containers_pause-amd64] + 5f70bf18a086: Mounted from netcat-hello-http + 41ff149e94f2: Pushed + 3.0: digest: sha256:796bf45888b8c74ead28547f4d3c7fdcde8c898fb67ef2b279186d7d7dbf72e8 size: 917 + + [vagrant@localhost ~]$ docker pull 10.64.33.81:5000/google_containers_pause-amd64:3.0 + Trying to pull repository 10.64.33.81:5000/google_containers_pause-amd64 ... + 3.0: Pulling from 10.64.33.81:5000/google_containers_pause-amd64 + a3ed95caeb02: Pull complete + f11233434377: Pull complete + Digest: sha256:796bf45888b8c74ead28547f4d3c7fdcde8c898fb67ef2b279186d7d7dbf72e8 + Status: Downloaded newer image for 10.64.33.81:5000/google_containers_pause-amd64:3.0 + + [vagrant@localhost ~]$ docker tag 10.64.33.81:5000/google_containers_pause-amd64:3.0 gcr.io/google_containers/pause-amd64:3.0 diff --git a/hack/kubernetes-clearwater/kubernetes/dex/dex.yaml b/hack/kubernetes-clearwater/kubernetes/dex/dex.yaml new file mode 100755 index 0000000..5b38a73 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/dex/dex.yaml @@ -0,0 +1,106 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: dex + name: dex +spec: + replicas: 3 + template: + metadata: + labels: + app: dex + spec: + containers: + - image: quay.io/coreos/dex:v2.0.0-beta.1 + name: dex + command: ["/usr/local/bin/dex", "serve", "/etc/dex/cfg/config.yaml"] + + ports: + - name: https + containerPort: 5556 + + volumeMounts: + - name: config + mountPath: /etc/dex/cfg + - name: tls + mountPath: /etc/dex/tls + + env: + - name: GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: github-client + key: client-id + - name: GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: github-client + key: client-secret + volumes: + - name: config + configMap: + name: dex + items: + - key: config.yaml + path: config.yaml + - name: tls + secret: + secretName: dex.example.com.tls +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: dex +data: + config.yaml: | + issuer: https://dex.example.com:32000 + storage: + type: kubernetes + config: + inCluster: true + web: + https: 0.0.0.0:5556 + tlsCert: /etc/dex/tls/tls.crt + tlsKey: /etc/dex/tls/tls.key + connectors: + - type: github + id: github + name: GitHub + config: + clientID: $GITHUB_CLIENT_ID + clientSecret: $GITHUB_CLIENT_SECRET + redirectURI: https://dex.example.com:32000/callback + org: kubernetes + oauth2: + skipApprovalScreen: true + + staticClients: + - id: example-app + redirectURIs: + - 'http://127.0.0.1:5555/callback' + name: 'Example App' + secret: ZXhhbXBsZS1hcHAtc2VjcmV0 + + enablePasswordDB: true + staticPasswords: + - email: "admin@example.com" + # bcrypt hash of the string "password" + hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" + username: "admin" + userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" +--- +apiVersion: v1 +kind: Service +metadata: + name: dex +spec: + type: NodePort + ports: + - name: dex + port: 5556 + protocol: TCP + targetPort: 5556 + nodePort: 32000 + selector: + app: dex \ No newline at end of file diff --git a/hack/kubernetes-clearwater/kubernetes/regenerate-certs.md b/hack/kubernetes-clearwater/kubernetes/regenerate-certs.md new file mode 100755 index 0000000..45ee398 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/regenerate-certs.md @@ -0,0 +1,83 @@ +* Prerequsites + +You have followed with above `kubernetes-installation` to create *KUBECONFIG* + +Stop legacy service instances + + [vagrant@master81 ~]$ for i in $(ls /etc/systemd/system/kube*); do svc=${i/\/etc\/systemd\/system\/}; sudo systemctl stop $svc; done + +* Certs and KUBECONFIG + +Generate certs + + [vagrant@master81 ~]$ vi salt-make-ca-cert.sh + + [vagrant@master81 ~]$ ./salt-make-ca-cert.sh + IP:10.64.33.81,IP:10.123.240.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local + Generate ca certs into /home/vagrant + + [vagrant@master81 ~]$ ls + bin kube kubecfg.key kubernetes.tar.gz server.cert + ca.crt kubecfg.crt kubernetes salt-make-ca-cert.sh server.key + + [vagrant@master81 ~]$ sudo cp ca.crt kubecfg.crt kubecfg.key server.cert server.key /srv/kubernetes/ + +Update `KUBECONFIG` + + [vagrant@master81 ~]$ encoded=$(sudo base64 -w0 /srv/kubernetes/ca.crt); kubectl config set clusters.kube.certificate-authority-data "$encoded" + + [vagrant@master81 ~]$ encoded=$(sudo base64 -w0 /srv/kubernetes/kubecfg.crt); kubectl config set users.admin.client-certificate-data "$encoded" + + [vagrant@master81 ~]$ encoded=$(sudo base64 -w0 /srv/kubernetes/kubecfg.key); kubectl config set users.admin.client-key-data "$encoded" + + [vagrant@master81 ~]$ sudo cp .kube/config /srv/kubernetes/kubeconfig + +Remove legacy DB + + [vagrant@master81 ~]$ etcdctl ls --recursive /registry/secrets + /registry/secrets/default + /registry/secrets/default/default-token-hs193 + /registry/secrets/kube-system + /registry/secrets/kube-system/default-token-6tjce + /registry/secrets/stackdocker + /registry/secrets/stackdocker/registry-tls + /registry/secrets/stackdocker/default-token-zr7w0 + /registry/secrets/stackdocker/registry-htpasswd + + [vagrant@master81 ~]$ etcdctl rm --recursive /registry/secrets/kube-system + [vagrant@master81 ~]$ etcdctl rm --recursive /registry/secrets/default + [vagrant@master81 ~]$ etcdctl rm /registry/secrets/stackdocker/default-token-zr7w0 + +Start service + + [vagrant@master81 ~]$ for i in $(ls /etc/systemd/system/kube*); do svc=${i/\/etc\/systemd\/system\/}; sudo systemctl start $svc; done + +Important !!! validate + + [vagrant@master81 ~]$ sudo journalctl --no-pager --pager-end --no-tail --unit=kube-apiserver.service + + [vagrant@master81 ~]$ sudo journalctl --no-pager --pager-end --no-tail --unit=kube-controller-manager.service + + [vagrant@master81 ~]$ sudo journalctl --no-pager --pager-end --no-tail --unit=kube-scheduler.service + + [vagrant@master81 ~]$ sudo journalctl --no-pager --pager-end --no-tail --unit=kubelet.service + + [vagrant@master81 ~]$ sudo journalctl --no-pager --pager-end --no-tail --unit=kube-proxy.service + + [vagrant@master81 ~]$ kubectl get all --all-namespaces + NAMESPACE NAME DESIRED CURRENT AGE + kube-system kubernetes-dashboard-v1.4.0 1 1 1h + NAMESPACE NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + default kubernetes 10.123.240.1 443/TCP 1d + default netcat-hello-http 10.123.242.12 80/TCP 1d + stackdocker registry 10.123.240.35 5000/TCP 17h + NAMESPACE NAME READY STATUS RESTARTS AGE + default netcat-hello-http-839099240-j628h 1/1 Running 9 1d + kube-system kubernetes-dashboard-v1.4.0-akswj 0/1 CrashLoopBackOff 15 1h + stackdocker registry-2499285598-9vdxa 1/1 Running 1 17h + +Important !!! update node and client with newest certs and kubeconfig + + [vagrant@localhost ~]$ scp 10.64.33.81:/home/vagrant/.kube/config .pki/kubernetes/master81/kubeconfig + vagrant@10.64.33.81's password: + config 100% 10KB 9.9KB/s 00:00 diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/README.md b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/README.md new file mode 100644 index 0000000..733bda3 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/README.md @@ -0,0 +1,47 @@ + +## Content + +### htpasswd + +Refer to https://github.com/docker/distribution/blob/v2.4.1/docs/deploying.md#native-basic-auth + +Example here is using *admin* as user name, and *password* as password + + [vagrant@master81 2.4.1]$ USERNAME=admin && PASSWORD=password && docker run --rm --entrypoint htpasswd docker.io/registry:2.4.1 -Bbn $USERNAME $PASSWORD > htpasswd + +Or + + [vagrant@master81 2.4.1]$ sudo yum install httpd-tools + + [vagrant@master81 2.4.1]$ htpasswd --help + Usage: + htpasswd [-cimBdpsDv] [-C cost] passwordfile username + htpasswd -b[cmBdpsDv] [-C cost] passwordfile username password + + htpasswd -n[imBdps] [-C cost] username + htpasswd -nb[mBdps] [-C cost] username password + -c Create a new file. + -n Don't update file; display results on stdout. + -b Use the password from the command line rather than prompting for it. + -i Read password from stdin without verification (for script usage). + -m Force MD5 encryption of the password (default). + -B Force bcrypt encryption of the password (very secure). + -C Set the computing time used for the bcrypt algorithm + (higher is more secure but slower, default: 5, valid: 4 to 31). + -d Force CRYPT encryption of the password (8 chars max, insecure). + -s Force SHA encryption of the password (insecure). + -p Do not encrypt the password (plaintext, insecure). + -D Delete the specified user. + -v Verify password for the specified user. + On other systems than Windows and NetWare the '-p' flag will probably not work. + The SHA algorithm does not use a salt and is less secure than the MD5 algorithm. + +Base64 + + [vagrant@master81 2.4.1]$ base64 -w0 htpasswd + YWRtaW46JDJ5JDA1JFZoblU4OFk0Lmo0WUh2QkV1c1JOMC45MkdzVnFjL244SWFCbXJCdVNNVXl5Zk96NWlaV2xpCgo= + +### tls + + + diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/config-configmap.yaml b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/config-configmap.yaml new file mode 100644 index 0000000..76977b5 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/config-configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-config + namespace: stackdocker +data: + config.yml: | + version: 0.1 + log: + fields: + service: registry + storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry + http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + health: + storagedriver: + enabled: true + interval: 30s + threshold: 3 + auth: + htpasswd: + realm: basic-realm + path: /auth/htpasswd diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/config.yml b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/config.yml new file mode 100644 index 0000000..1978aed --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/config.yml @@ -0,0 +1,22 @@ +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] +health: + storagedriver: + enabled: true + interval: 30s + threshold: 3 +auth: + htpasswd: + realm: basic-realm + path: /auth/htpasswd diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/htpasswd b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/htpasswd new file mode 100644 index 0000000..0178351 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/htpasswd @@ -0,0 +1,2 @@ +admin:$2y$05$JB5ETuLk81h2hHe9EJFdfOix8vGM93j4sY5BfkFLjGAc2fnf0rX8u + diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/htpasswd-secret.yaml b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/htpasswd-secret.yaml new file mode 100644 index 0000000..9bb84d5 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/htpasswd-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: registry-htpasswd + namespace: stackdocker +data: + htpasswd: YWRtaW46JDJ5JDA1JEpCNUVUdUxrODFoMmhIZTlFSkZkZk9peDh2R005M2o0c1k1QmZrRkxqR0FjMmZuZjByWDh1Cgo= +type: Opaque diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/registry-deployment.yaml b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/registry-deployment.yaml new file mode 100644 index 0000000..7f18dfd --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/registry-deployment.yaml @@ -0,0 +1,71 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: registry + namespace: stackdocker +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + stackdocker-app: docker-distribution + template: + metadata: + labels: + stackdocker-app: docker-distribution + spec: + containers: + - env: + - name: DOCKER_REGISTRY_CONFIG + value: /etc/docker/registry/config.yml + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /certs/tls.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /certs/tls.key + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + image: docker.io/registry:2.4.1 + name: registry + ports: + - containerPort: 5000 + hostIP: 10.64.33.81 + hostPort: 5000 + securityContext: + privileged: true + volumeMounts: + - name: basic-realm + mountPath: /auth + readOnly: true + - name: selfsigned-tls + mountPath: /certs + readOnly: true + - name: config + mountPath: /etc/docker/registry + readOnly: true + - name: repo + mountPath: /var/lib/registry + volumes: + - name: basic-realm + secret: + secretName: registry-htpasswd + items: + - key: htpasswd + path: htpasswd + - name: selfsigned-tls + secret: + secretName: registry-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + - name: config + configMap: + name: registry-config + items: + - key: config.yml + path: config.yml + - name: repo + hostPath: + path: /var/lib/registry diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/registry-service.yaml b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/registry-service.yaml new file mode 100644 index 0000000..ba68e2c --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/registry-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: registry + namespace: stackdocker +spec: + clusterIP: 10.123.240.35 + ports: + - port: 5000 + selector: + stackdocker-app: docker-distribution diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/stackdocker-namespace.yaml b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/stackdocker-namespace.yaml new file mode 100644 index 0000000..40823ee --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/stackdocker-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: stackdocker diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls-secret.yaml b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls-secret.yaml new file mode 100644 index 0000000..8393f40 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: registry-tls + namespace: stackdocker +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURpVENDQW5HZ0F3SUJBZ0lRYnVoK24vSGI3TFVxMlZpVGd6ckkyakFOQmdrcWhraUc5dzBCQVFzRkFEQXgKTVJRd0VnWURWUVFLRXd0TWIyY2dRMjkxY21sbGNqRVpNQmNHQTFVRUF4TVFNVEF1TmpRdU16TXVPREU2TlRBdwpNREFlRncweE5qRXhNVFF5TURRME1UQmFGdzB4T1RBNE1URXlNRFEwTVRCYU1ERXhGREFTQmdOVkJBb1RDMHh2Clp5QkRiM1Z5YVdWeU1Sa3dGd1lEVlFRREV4QXhNQzQyTkM0ek15NDRNVG8xTURBd01JSUJJakFOQmdrcWhraUcKOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXpuTEhwM29RcmI2TGpFK2piOUhhQkFpaUFzR21vVWxmbTcxegplZ0l1dlV5Q0hCNTJiZjUraE1tdSszWEY3VjArNTRRSUY3VmxXM2NmZURNT0FKdUpyNUxSdE5IVThUTWJZVTliCjE0emR1MjRqQ0IzWFJrTFBhQThkV2NidW11RSs3ME13Q0dLNUFkSUx1dFJZQzIzV01SYWJNcCtZYU9OVGlSSTYKbWpaVW0zenVZZlZhaDJNTU43YkEyMXhGVzFuUmdndkNiM0w1Mkx6TW1BZG5pYjZuQ2tUNUZESGhRM2V3MkpyTQowZk02dnEyT3VXOWk3NS9tREl5YTN0dUNLdm1YVVp3a045clY4eUJIakl4Ym9MMWV4NTRENy9DY0Z1MjdjRExYCnpERE4zVUlCQlJIRHhMemZZMUZEL2RYZmtjK2wwT24yV01Ha256ajFNQUNVM3h3b1J3SURBUUFCbzRHY01JR1oKTUE0R0ExVWREd0VCL3dRRUF3SUNwREFUQmdOVkhTVUVEREFLQmdnckJnRUZCUWNEQVRBUEJnTlZIUk1CQWY4RQpCVEFEQVFIL01HRUdBMVVkRVFSYU1GaUNJbkpsWjJsemRISjVMbVJsWm1GMWJIUXVjM1pqTG1Oc2RYTjBaWEl1CmJHOWpZV3lDSm5KbFoybHpkSEo1TG5OMFlXTnJaRzlqYTJWeUxuTjJZeTVqYkhWemRHVnlMbXh2WTJGc2h3UUsKUUNGUmh3UUtlL0FqTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFETmh0VmszRURoZm11bjdSbml0bW9uSVJTUgpuejJ6SDRwcHpmWW0yT2kya0hKLzBkU3hHeDRKOFV4RmtSdnYvWlRidE1kbVlSanEzK3ZQd3VSb3dlVVZPQXlnCks4NUg5dHNUNXQyRzhaYzZaV3U1S25ZNlBmTmNYck1rV1BXR0JzWUhvQVNGOXZEamVqbEh4UHdqRFVDRlNrNHYKU0V4cE1WdnpTUHpla1puWWpOVXhOQVd3RDFDWDF4dnVtNExad2ZGMUg3bjI4dCtlQ1pBaUtYNEVwNzVGZHhDaAp2SG5jUUpJZ0pzYXZBUUtCcElCM2h1aHVsYk5aamd1b3R0QWJNV0Zob0U5QnBlcHF3L2FzVmluUTZUdml6STdiCkp1WTE2SmtWSHBqOHZsenhDVHlKbVhTOVlLUndKTlRGOEtWM1oydUR6N0VoS29NbnZLUkxyVlVvcDNmcQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBem5MSHAzb1FyYjZMakUramI5SGFCQWlpQXNHbW9VbGZtNzF6ZWdJdXZVeUNIQjUyCmJmNStoTW11KzNYRjdWMCs1NFFJRjdWbFczY2ZlRE1PQUp1SnI1TFJ0TkhVOFRNYllVOWIxNHpkdTI0akNCM1gKUmtMUGFBOGRXY2J1bXVFKzcwTXdDR0s1QWRJTHV0UllDMjNXTVJhYk1wK1lhT05UaVJJNm1qWlVtM3p1WWZWYQpoMk1NTjdiQTIxeEZXMW5SZ2d2Q2IzTDUyTHpNbUFkbmliNm5Da1Q1RkRIaFEzZXcySnJNMGZNNnZxMk91VzlpCjc1L21ESXlhM3R1Q0t2bVhVWndrTjlyVjh5QkhqSXhib0wxZXg1NEQ3L0NjRnUyN2NETFh6REROM1VJQkJSSEQKeEx6ZlkxRkQvZFhma2MrbDBPbjJXTUdrbnpqMU1BQ1UzeHdvUndJREFRQUJBb0lCQUhpUksreXhLd0J0SVk1bQpEQ3NId1cxTnBXQTZ2QmZCaENhdU03eHpmZHhyekNpajJjSlBPSkJRUVBMMlN3dGV3NWZ0MGRRV3lnVHF3dVF0CjNPV1o4SVVyRjNHa2dkb0RZT0FwdXBkZDJ6alZEM3gwQVA4MWtXOVNORy96Ny9GZnpxTm85NnBPRFlIMHE0a0cKN2d4dXJ3S29FeWN6Y1kwQUJnOHRLREJidm5tM0pUWGwwMXRpL2JmQ29uWDlJR0lwcm9xbTlGQklCd0J5ZzJCegpxTlp1RXcvNTFKYlBSMmp5cWZuZG5HY3BTdGU2S3BpM2UxUEpEd1hHdUV1ZGl1LzI2R2xyckgyVkpWZXNDS1lYCnpBRE8zT1NVU0Z5VVkveUxGOHlXekZOYkpZOEVVcE9ONWVjRldmc1BEK0E1Szh6RGFrc0ZiZkkwTjdsZmhzbUIKd1B0azZCRUNnWUVBOTVyNGNwc0loMW5keUJxUzVqcE13cmxISzNub1VJZnJtT3B2eDVNeUxEWjlrME9nVDJZSQpNeWw2SUEvd0NTb3FDQTRkUVp3dkhEV2hKWm4rTGZ0TlFScm5LK2h4OVRncGJvN3h3Wm95UzQzOW1EMURsN3lTCmxsdWgzMWNaNHYyT296OW1Nek5OSTBqOTRPS0hRNlJUbGNzUzVJdnIwdHRheWF2RFBnY2diQXNDZ1lFQTFYS1kKOFdMZm5YTHJ4MUh1cWpHSUJEbzF6dDRoRkdrZERVclpHUWJCZjBTbVpHNjk2ZkJxMUZvQzE0R2QxWnJaQzJSawo3TVdQSXM1T2VkQ2s4SzBqcFUrZ3pEWGYvdkdxKzdRbGhnWEdaRWdOaE9HcVBuOWRGUTU1eEsrOUl3UlhkNnZaClZQUDZaWHVIOUJFc2FxL0JBOStWUmh1SVNSUlBaTTNuaFdPRm5qVUNnWUVBNGxMRXhmenZ6c2hWOEpKWnRCZVcKRW1kSXBuVEg1TlJZaFczRXZxZk5OSnlhTEFHa2h1NjhrQWRabWZUM2Mya1hBa1hrZUJSSyszNEp6bFQwQVJPegppYzhOZnhUNUZTeEFzczMyTFFERStFclJnbFo5Y2pUcnlJUDVjaXdyZlZCSEVQRGllZXBxbDlZZ09XN1dOUDMvCmFPdzJXTEdNM1MranpMTXp3MzdBSkFFQ2dZQXJWVVdaaytJaisrWEVKRzBZbmsrbjFBMkNmdlFtU2FTOVF4bksKQytZU3FXUkNkWDBXMWxsSjNUQkFnaThCNDVyMnhoMzdONngwZTMvUkdUdXpKT05jdDByRXlwYzhFZUlOa2FnZwppbUdaa0VzMlUvNFZEQlJncHZ5dm9xSDZUdkpXK0V0NkJNSXNBaVU2dHBYVzN2R0hrTWVrWVFpTkZaTjh1cFNyCkx1WThsUUtCZ1FDa2t1LzBXUUxXQ2I5T0t5aGJDamdwU2FoMlJUNzhwSnIrQ1JHNGJlQjZkYnZRS3BBK3F0b3AKR3hrL3VseUhVd3BpUHptWGh3ZitWR3VBVUNML2V1TGpaMWR0UWtBZCtvbTVMQ2E1THMrZGxVU0YxWkFiZTRwegpLbFlRMXRUVm81cUhENWd6Z2hURWdZdlpUMTFlWjlVVHA2WjdxOTdIa2VlMFFlVDFGd2tKNWc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls.crt b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls.crt new file mode 100644 index 0000000..5db7aa0 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIQbuh+n/Hb7LUq2ViTgzrI2jANBgkqhkiG9w0BAQsFADAx +MRQwEgYDVQQKEwtMb2cgQ291cmllcjEZMBcGA1UEAxMQMTAuNjQuMzMuODE6NTAw +MDAeFw0xNjExMTQyMDQ0MTBaFw0xOTA4MTEyMDQ0MTBaMDExFDASBgNVBAoTC0xv +ZyBDb3VyaWVyMRkwFwYDVQQDExAxMC42NC4zMy44MTo1MDAwMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAznLHp3oQrb6LjE+jb9HaBAiiAsGmoUlfm71z +egIuvUyCHB52bf5+hMmu+3XF7V0+54QIF7VlW3cfeDMOAJuJr5LRtNHU8TMbYU9b +14zdu24jCB3XRkLPaA8dWcbumuE+70MwCGK5AdILutRYC23WMRabMp+YaONTiRI6 +mjZUm3zuYfVah2MMN7bA21xFW1nRggvCb3L52LzMmAdnib6nCkT5FDHhQ3ew2JrM +0fM6vq2OuW9i75/mDIya3tuCKvmXUZwkN9rV8yBHjIxboL1ex54D7/CcFu27cDLX +zDDN3UIBBRHDxLzfY1FD/dXfkc+l0On2WMGknzj1MACU3xwoRwIDAQABo4GcMIGZ +MA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8E +BTADAQH/MGEGA1UdEQRaMFiCInJlZ2lzdHJ5LmRlZmF1bHQuc3ZjLmNsdXN0ZXIu +bG9jYWyCJnJlZ2lzdHJ5LnN0YWNrZG9ja2VyLnN2Yy5jbHVzdGVyLmxvY2FshwQK +QCFRhwQKe/AjMA0GCSqGSIb3DQEBCwUAA4IBAQDNhtVk3EDhfmun7RnitmonIRSR +nz2zH4ppzfYm2Oi2kHJ/0dSxGx4J8UxFkRvv/ZTbtMdmYRjq3+vPwuRoweUVOAyg +K85H9tsT5t2G8Zc6ZWu5KnY6PfNcXrMkWPWGBsYHoASF9vDjejlHxPwjDUCFSk4v +SExpMVvzSPzekZnYjNUxNAWwD1CX1xvum4LZwfF1H7n28t+eCZAiKX4Ep75FdxCh +vHncQJIgJsavAQKBpIB3huhulbNZjguottAbMWFhoE9Bpepqw/asVinQ6TvizI7b +JuY16JkVHpj8vlzxCTyJmXS9YKRwJNTF8KV3Z2uDz7EhKoMnvKRLrVUop3fq +-----END CERTIFICATE----- diff --git a/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls.key b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls.key new file mode 100644 index 0000000..1a8e5bf --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/registry/2.4.1/tls.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAznLHp3oQrb6LjE+jb9HaBAiiAsGmoUlfm71zegIuvUyCHB52 +bf5+hMmu+3XF7V0+54QIF7VlW3cfeDMOAJuJr5LRtNHU8TMbYU9b14zdu24jCB3X +RkLPaA8dWcbumuE+70MwCGK5AdILutRYC23WMRabMp+YaONTiRI6mjZUm3zuYfVa +h2MMN7bA21xFW1nRggvCb3L52LzMmAdnib6nCkT5FDHhQ3ew2JrM0fM6vq2OuW9i +75/mDIya3tuCKvmXUZwkN9rV8yBHjIxboL1ex54D7/CcFu27cDLXzDDN3UIBBRHD +xLzfY1FD/dXfkc+l0On2WMGknzj1MACU3xwoRwIDAQABAoIBAHiRK+yxKwBtIY5m +DCsHwW1NpWA6vBfBhCauM7xzfdxrzCij2cJPOJBQQPL2Swtew5ft0dQWygTqwuQt +3OWZ8IUrF3GkgdoDYOApupdd2zjVD3x0AP81kW9SNG/z7/FfzqNo96pODYH0q4kG +7gxurwKoEyczcY0ABg8tKDBbvnm3JTXl01ti/bfConX9IGIproqm9FBIBwByg2Bz +qNZuEw/51JbPR2jyqfndnGcpSte6Kpi3e1PJDwXGuEudiu/26GlrrH2VJVesCKYX +zADO3OSUSFyUY/yLF8yWzFNbJY8EUpON5ecFWfsPD+A5K8zDaksFbfI0N7lfhsmB +wPtk6BECgYEA95r4cpsIh1ndyBqS5jpMwrlHK3noUIfrmOpvx5MyLDZ9k0OgT2YI +Myl6IA/wCSoqCA4dQZwvHDWhJZn+LftNQRrnK+hx9Tgpbo7xwZoyS439mD1Dl7yS +lluh31cZ4v2Ooz9mMzNNI0j94OKHQ6RTlcsS5Ivr0ttayavDPgcgbAsCgYEA1XKY +8WLfnXLrx1HuqjGIBDo1zt4hFGkdDUrZGQbBf0SmZG696fBq1FoC14Gd1ZrZC2Rk +7MWPIs5OedCk8K0jpU+gzDXf/vGq+7QlhgXGZEgNhOGqPn9dFQ55xK+9IwRXd6vZ +VPP6ZXuH9BEsaq/BA9+VRhuISRRPZM3nhWOFnjUCgYEA4lLExfzvzshV8JJZtBeW +EmdIpnTH5NRYhW3EvqfNNJyaLAGkhu68kAdZmfT3c2kXAkXkeBRK+34JzlT0AROz +ic8NfxT5FSxAss32LQDE+ErRglZ9cjTryIP5ciwrfVBHEPDieepql9YgOW7WNP3/ +aOw2WLGM3S+jzLMzw37AJAECgYArVUWZk+Ij++XEJG0Ynk+n1A2CfvQmSaS9QxnK +C+YSqWRCdX0W1llJ3TBAgi8B45r2xh37N6x0e3/RGTuzJONct0rEypc8EeINkagg +imGZkEs2U/4VDBRgpvyvoqH6TvJW+Et6BMIsAiU6tpXW3vGHkMekYQiNFZN8upSr +LuY8lQKBgQCkku/0WQLWCb9OKyhbCjgpSah2RT78pJr+CRG4beB6dbvQKpA+qtop +Gxk/ulyHUwpiPzmXhwf+VGuAUCL/euLjZ1dtQkAd+om5LCa5Ls+dlUSF1ZAbe4pz +KlYQ1tTVo5qHD5gzghTEgYvZT11eZ9UTp6Z7q97Hkee0QeT1FwkJ5g== +-----END RSA PRIVATE KEY----- diff --git a/hack/kubernetes-clearwater/kubernetes/release-install.md b/hack/kubernetes-clearwater/kubernetes/release-install.md new file mode 100755 index 0000000..3c566f3 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/release-install.md @@ -0,0 +1,88 @@ +Download, for example v1.3.7 + + [vagrant@localhost ~]$ wget -c https://github.com/kubernetes/kubernetes/releases/download/v1.3.7/kubernetes.tar.gz + --2016-11-15 10:10:01-- https://github.com/kubernetes/kubernetes/releases/download/v1.3.7/kubernetes.tar.gz + Resolving github.com (github.com)... 192.30.253.113, 192.30.253.112 + Connecting to github.com (github.com)|192.30.253.113|:443... connected. + HTTP request sent, awaiting response... 302 Found + Location: https://github-cloud.s3.amazonaws.com/releases/20580498/f5d7ff58-790a-11e6-84ac-2741d0fb18f7.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAISTNZFOVBIJMK3TQ%2F20161115%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20161115T101002Z&X-Amz-Expires=300&X-Amz-Signature=b7034ab7decfe8d61a88e0561a42e9ab6fd339d3027021c54b7b22f4cd3ae6e1&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Dkubernetes.tar.gz&response-content-type=application%2Foctet-stream [following] + --2016-11-15 10:10:03-- https://github-cloud.s3.amazonaws.com/releases/20580498/f5d7ff58-790a-11e6-84ac-2741d0fb18f7.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAISTNZFOVBIJMK3TQ%2F20161115%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20161115T101002Z&X-Amz-Expires=300&X-Amz-Signature=b7034ab7decfe8d61a88e0561a42e9ab6fd339d3027021c54b7b22f4cd3ae6e1&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Dkubernetes.tar.gz&response-content-type=application%2Foctet-stream + Resolving github-cloud.s3.amazonaws.com (github-cloud.s3.amazonaws.com)... 52.216.224.232 + Connecting to github-cloud.s3.amazonaws.com (github-cloud.s3.amazonaws.com)|52.216.224.232|:443... connected. + HTTP request sent, awaiting response... 200 OK + Length: 1489843499 (1.4G) [application/octet-stream] + Saving to: 鈥榢ubernetes.tar.gz鈥▒ + + 0% [ ] 12,924,489 286KB/s eta 84m 1s + +Stop legacy service instance + + [vagrant@localhost ~]$ for i in $(ls /etc/systemd/system/kube*); do svc=${i/\/etc\/systemd\/system\/}; sudo systemctl stop $svc; done + +Extract + + [vagrant@localhost ~]$ tar -tzf kubernetes.tar.gz + + kubernetes/server/ + kubernetes/server/kubernetes-salt.tar.gz + + kubernetes/server/kubernetes-manifests.tar.gz + + kubernetes/server/kubernetes-server-linux-amd64.tar.gz + + [vagrant@localhost ~]$ sudo tar -C /opt -zxf kubernetes.tar.gz kubernetes/server/kubernetes-server-linux-amd64.tar.gz + + [vagrant@localhost ~]$ cd /opt/kubernetes/ + + [vagrant@localhost kubernetes]$ sudo mv server/bin 1.3.10/release + + [vagrant@localhost ~]$ sudo tar -C /opt -zxf /opt/kubernetes/server/kubernetes-server-linux-amd64.tar.gz + +Validate + + [vagrant@localhost kubernetes]$ /opt/kubernetes/server/bin/kubelet --version + Kubernetes v1.3.7 + +Start fresh service instance + + [vagrant@localhost kubernetes]$ for i in $(ls /etc/systemd/system/kube*); do svc=${i/\/etc\/systemd\/system\/}; sudo systemctl start $svc; done + +Important !!! to validate service running + + [vagrant@localhost kubernetes]$ sudo journalctl --no-tail --no-pager --pager-end -u kube-apiserver.service + + [vagrant@localhost kubernetes]$ sudo journalctl --no-tail --no-pager --pager-end -u kube-controller-manager.service + E1115 11:33:42.038303 14523 controllermanager.go:253] Failed to start service controller: ServiceController should not be run without a cloudprovider. + E1115 11:33:42.606684 14523 util.go:45] Metric for serviceaccount_controller already registered + + [vagrant@localhost kubernetes]$ sudo journalctl --no-tail --no-pager --pager-end -u kube-scheduler.service + + [vagrant@localhost kubernetes]$ sudo journalctl --no-tail --no-pager --pager-end -u kubelet.service + E1115 11:33:43.464190 14538 kubelet.go:954] Image garbage collection failed: unable to find data for container / + E1115 11:33:43.628377 14538 kubelet.go:2397] Failed to check if disk space is available for the runtime: failed to get fs info for "runtime": unable to find data for container / + E1115 11:33:43.628393 14538 kubelet.go:2405] Failed to check if disk space is available on the root partition: failed to get fs info for "root": unable to find data for container / + E1115 11:33:43.629611 14538 manager.go:240] Registration of the rkt container factory failed: unable to communicate with Rkt api service: rkt: cannot tcp Dial rkt api service: dial tcp [::1]:15441: getsockopt: connection refused + + [vagrant@localhost kubernetes]$ sudo journalctl --no-tail --no-pager --pager-end -u kube-proxy.service + + [vagrant@localhost kubernetes]$ kubectl get ep,svc + NAME ENDPOINTS AGE + kubernetes 10.64.33.81:6443 1d + netcat-hello-http 10.121.24.3:80 1d + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kubernetes 10.123.240.1 443/TCP 1d + netcat-hello-http 10.123.242.12 80/TCP 1d + + [vagrant@localhost kubernetes]$ curl 10.121.24.3 + welcome

hello world

+ + [vagrant@localhost kubernetes]$ curl 10.123.242.12 + welcome

hello world

+ + [vagrant@localhost kubernetes]$ docker push 10.64.33.81:5000/netcat-hello-http + The push refers to a repository [10.64.33.81:5000/netcat-hello-http] + 5f70bf18a086: Layer already exists + 14996164fa4d: Layer already exists + 000ac623049f: Layer already exists + 3f50a8137adf: Layer already exists + latest: digest: sha256:e21c11edf699f9c582b46388f269a44ee5f0662b099cca7cfd30d5c6970cf563 size: 1748 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/README.md b/hack/kubernetes-clearwater/kubernetes/saltbase/README.md new file mode 100644 index 0000000..8bc506d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/README.md @@ -0,0 +1,20 @@ +# SaltStack configuration + +This is the root of the SaltStack configuration for Kubernetes. A high +level overview for the Kubernetes SaltStack configuration can be found [in the docs tree.](../../docs/admin/salt.md) + +This SaltStack configuration currently applies to default +configurations for Debian-on-GCE, Fedora-on-Vagrant, and Ubuntu-on-AWS. +(That doesn't mean it can't be made to apply to an +arbitrary configuration, but those are only the in-tree OS/IaaS +combinations supported today.) As you peruse the configuration, these +are shorthanded as `gce`, `vagrant`, `aws` in `grains.cloud`; +the documentation in this tree uses this same shorthand for convenience. + +See more: +* [pillar](pillar/) +* [reactor](reactor/) +* [salt](salt/) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/saltbase/README.md?pixel)]() diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/install.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/install.sh new file mode 100755 index 0000000..b567909 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/install.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will set up the salt directory on the target server. It takes one +# argument that is a tarball with the pre-compiled kubernetes server binaries. + +set -o errexit +set -o nounset +set -o pipefail + +SALT_ROOT=$(dirname "${BASH_SOURCE}") +readonly SALT_ROOT + +readonly KUBE_DOCKER_WRAPPED_BINARIES=( + kube-apiserver + kube-controller-manager + kube-scheduler + kube-proxy +) + +readonly SERVER_BIN_TAR=${1-} +if [[ -z "$SERVER_BIN_TAR" ]]; then + echo "!!! No binaries specified" + exit 1 +fi + +# Create a temp dir for untaring +KUBE_TEMP=$(mktemp --tmpdir=/srv -d -t kubernetes.XXXXXX) +trap 'rm -rf "${KUBE_TEMP}"' EXIT + +# This file is meant to run on the master. It will install the salt configs +# into the appropriate place on the master. We do this by creating a new set of +# salt trees and then quickly mv'ing them where the old ones were. + +readonly SALTDIRS=(salt pillar reactor) + +echo "+++ Installing salt files into new trees" +rm -rf /srv/salt-new +mkdir -p /srv/salt-new + +# This bash voodoo will prepend $SALT_ROOT to the start of each item in the +# $SALTDIRS array +cp -v -R --preserve=mode "${SALTDIRS[@]/#/${SALT_ROOT}/}" /srv/salt-new + +echo "+++ Installing salt overlay files" +for dir in "${SALTDIRS[@]}"; do + if [[ -d "/srv/salt-overlay/$dir" ]]; then + cp -v -R --preserve=mode "/srv/salt-overlay/$dir" "/srv/salt-new/" + fi +done + +echo "+++ Install binaries from tar: $1" +tar -xz -C "${KUBE_TEMP}" -f "$1" +mkdir -p /srv/salt-new/salt/kube-bins +mkdir -p /srv/salt-new/salt/kube-addons-images +mkdir -p /srv/salt-new/salt/kube-docs +cp -v "${KUBE_TEMP}/kubernetes/server/bin/"* /srv/salt-new/salt/kube-bins/ +cp -v "${KUBE_TEMP}/kubernetes/addons/"* /srv/salt-new/salt/kube-addons-images/ +cp -v "${KUBE_TEMP}/kubernetes/LICENSES" /srv/salt-new/salt/kube-docs/ + +kube_bin_dir="/srv/salt-new/salt/kube-bins"; +docker_images_sls_file="/srv/salt-new/pillar/docker-images.sls"; +for docker_file in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do + docker_tag=$(cat ${kube_bin_dir}/${docker_file}.docker_tag); + if [[ ! -z "${KUBE_IMAGE_TAG:-}" ]]; then + docker_tag="${KUBE_IMAGE_TAG}" + fi + sed -i "s/#${docker_file}_docker_tag_value#/${docker_tag}/" "${docker_images_sls_file}"; +done + +cat <>"${docker_images_sls_file}" +kube_docker_registry: '$(echo ${KUBE_DOCKER_REGISTRY:-gcr.io/google_containers})' +EOF + +# TODO(zmerlynn): Forgive me, this is really gross. But in order to +# avoid breaking the non-Salt deployments, which already painfully +# have to templatize a couple of the add-ons anyways, manually +# templatize the addon registry for regional support. When we get +# better templating, we can fix this. +readonly kube_addon_registry="${KUBE_ADDON_REGISTRY:-gcr.io/google_containers}" +if [[ "${kube_addon_registry}" != "gcr.io/google_containers" ]]; then + find /srv/salt-new -name \*.yaml -or -name \*.yaml.in | \ + xargs sed -ri "s@(image:\s.*)gcr.io/google_containers@\1${kube_addon_registry}@" + # All the legacy .manifest files with hardcoded gcr.io are JSON. + find /srv/salt-new -name \*.manifest -or -name \*.json | \ + xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" +fi + +echo "+++ Swapping in new configs" +for dir in "${SALTDIRS[@]}"; do + if [[ -d "/srv/$dir" ]]; then + rm -rf "/srv/$dir" + fi + mv -v "/srv/salt-new/$dir" "/srv/$dir" +done + +rm -rf /srv/salt-new diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/README.md b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/README.md new file mode 100644 index 0000000..3b09c89 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/README.md @@ -0,0 +1,22 @@ +The +[SaltStack pillar](http://docs.saltstack.com/en/latest/topics/pillar/) +data is partially statically derived from the contents of this +directory. The bulk of the pillars are hard to perceive from browsing +this directory, though, because they are written into +[cluster-params.sls](cluster-params.sls) at cluster inception. + +* [cluster-params.sls](cluster-params.sls) is generated entirely at cluster inception. See e.g. [configure-vm.sh](../../gce/configure-vm.sh#L262) +* [docker-images.sls](docker-images.sls) stores the Docker tags of the current Docker-wrapped server binaries, twiddling by the Salt install script +* [logging.sls](logging.sls) defines the cluster log level +* [mine.sls](mine.sls): defines the variables shared across machines in the Salt + mine. It is starting to be largely deprecated in use, and is totally + unavailable on GCE, which runs standalone. +* [privilege.sls](privilege.sls) defines whether privileged containers are allowed. +* [top.sls](top.sls) defines which pillars are active across the cluster. + +## Future work + +Document the current pillars across providers + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/saltbase/pillar/README.md?pixel)]() diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/cluster-params.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/cluster-params.sls new file mode 100644 index 0000000..3f743b4 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/cluster-params.sls @@ -0,0 +1,4 @@ +# This file is meant to be replaced with cluster specific parameters if necessary. + +# Examples: +# node_instance_prefix: diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/docker-images.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/docker-images.sls new file mode 100644 index 0000000..6425691 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/docker-images.sls @@ -0,0 +1,5 @@ +# This file is populated when kubernetes is built. +kube-apiserver_docker_tag: #kube-apiserver_docker_tag_value# +kube-controller-manager_docker_tag: #kube-controller-manager_docker_tag_value# +kube-scheduler_docker_tag: #kube-scheduler_docker_tag_value# +kube-proxy_docker_tag: #kube-proxy_docker_tag_value# diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/logging.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/logging.sls new file mode 100644 index 0000000..25e8351 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/logging.sls @@ -0,0 +1 @@ +log_level: "--v=2" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/mine.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/mine.sls new file mode 100644 index 0000000..45035e4 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/mine.sls @@ -0,0 +1,12 @@ +{% if grains.cloud is defined and grains.cloud == 'gce' -%} +# On GCE, there is no Salt mine. We run standalone. +{% else %} +# Allow everyone to see cached values of who sits at what IP +{% set networkInterfaceName = "eth0" %} +{% if grains.networkInterfaceName is defined %} + {% set networkInterfaceName = grains.networkInterfaceName %} +{% endif %} +mine_functions: + network.ip_addrs: [{{networkInterfaceName}}] + grains.items: [] +{% endif -%} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/privilege.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/privilege.sls new file mode 100644 index 0000000..caa1549 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/privilege.sls @@ -0,0 +1,2 @@ +# If true, allow privileged containers to be created by API +allow_privileged: true diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/systemd.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/systemd.sls new file mode 100644 index 0000000..ee84738 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/systemd.sls @@ -0,0 +1,9 @@ +{% if grains['oscodename'] in [ 'vivid', 'wily', 'jessie' ] %} +is_systemd: True +systemd_system_path: /lib/systemd/system +{% elif grains['os_family'] == 'RedHat' %} +is_systemd: True +systemd_system_path: /usr/lib/systemd/system +{% else %} +is_systemd: False +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/top.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/top.sls new file mode 100644 index 0000000..d9ad7db --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/pillar/top.sls @@ -0,0 +1,8 @@ +base: + '*': + - mine + - cluster-params + - logging + - docker-images + - privilege + - systemd diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/README.md b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/README.md new file mode 100644 index 0000000..2b20c47 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/README.md @@ -0,0 +1,6 @@ +[SaltStack reactor](http://docs.saltstack.com/en/latest/topics/reactor/) files, largely defining reactions to new nodes. + +**Ignored for GCE, which runs standalone on each machine** + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/saltbase/reactor/README.md?pixel)]() diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-masters.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-masters.sls new file mode 100644 index 0000000..246f7eb --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-masters.sls @@ -0,0 +1,10 @@ +# This runs highstate on the master node(s). +# +# Some of the cluster deployment scripts pass the list of minion addresses to +# the apiserver as a command line argument. This list needs to be updated if a +# new minion is started, so run highstate on the master(s) when this happens. +# +highstate_master: + cmd.state.highstate: + - tgt: 'roles:kubernetes-master' + - expr_form: grain diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-minions.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-minions.sls new file mode 100644 index 0000000..7bd00d0 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-minions.sls @@ -0,0 +1,11 @@ +# This runs highstate on the minion nodes. +# +# Some of the cluster deployment scripts use the list of minions on the minions +# themselves (for example: every minion is configured with static routes to +# every other minion on a vSphere deployment). To propagate changes throughout +# the pool, run highstate on all minions whenever a single minion starts. +# +highstate_minions: + cmd.state.highstate: + - tgt: 'roles:kubernetes-pool' + - expr_form: grain diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-new.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-new.sls new file mode 100644 index 0000000..825749d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/reactor/highstate-new.sls @@ -0,0 +1,4 @@ +# This runs highstate only on the NEW node, regardless of type. +highstate_new: + cmd.state.highstate: + - tgt: {{ data['id'] }} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/README.md b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/README.md new file mode 100644 index 0000000..0f9879b --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/README.md @@ -0,0 +1,32 @@ +This directory forms the base of the main SaltStack configuration. The +place to start with any SaltStack configuration is +[top.sls](top.sls). However, unless you are particularly keen on +reading Jinja templates, the following tables break down what +configurations run on what providers. (NB: The [_states](_states/) +directory is a special directory included by Salt for `ensure` blocks, +and is only used for the [docker](docker/) config.) + +Key: M = Config applies to master, n = config applies to nodes + +Config | GCE | Vagrant | AWS | +----------------------------------------------------|-------|---------|-----| +[debian-auto-upgrades](debian-auto-upgrades/) | M n | M n | M n | +[docker](docker/) | M n | M n | M n | +[etcd](etcd/) | M | M | M | +[fluentd-es](fluentd-es/) (pillar conditional) | M n | M n | M n | +[fluentd-gcp](fluentd-gcp/) (pillar conditional) | M n | M n | M n | +[generate-cert](generate-cert/) | M | M | M | +[kube-addons](kube-addons/) | M | M | M | +[kube-apiserver](kube-apiserver/) | M | M | M | +[kube-controller-manager](kube-controller-manager/) | M | M | M | +[kube-proxy](kube-proxy/) | n | n | n | +[kube-scheduler](kube-scheduler/) | M | M | M | +[kubelet](kubelet/) | M n | M n | M n | +[logrotate](logrotate/) | M n | n | M n | +[supervisord](supervisor/) | M n | M n | M n | +[static-routes](static-routes/) (vsphere only) | | | | +[base](base.sls) | M n | M n | M n | +[kube-client-tools](kube-client-tools.sls) | M | M | M | + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/saltbase/salt/README.md?pixel)]() diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/base.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/base.sls new file mode 100644 index 0000000..a8eac1e --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/base.sls @@ -0,0 +1,37 @@ +pkg-core: + pkg.installed: + - names: + - curl +{% if grains['os_family'] == 'RedHat' %} + - python + - git +{% else %} + - apt-transport-https + - python-apt + - nfs-common + - socat +{% endif %} +# Ubuntu installs netcat-openbsd by default, but on GCE/Debian netcat-traditional is installed. +# They behave slightly differently. +# For sanity, we try to make sure we have the same netcat on all OSes (#15166) +{% if grains['os'] == 'Ubuntu' %} + - netcat-traditional +{% endif %} +# Make sure git is installed for mounting git volumes +{% if grains['os'] == 'Ubuntu' %} + - git +{% endif %} + +/usr/local/share/doc/kubernetes: + file.directory: + - user: root + - group: root + - mode: 755 + - makedirs: True + +/usr/local/share/doc/kubernetes/LICENSES: + file.managed: + - source: salt://kube-docs/LICENSES + - user: root + - group: root + - mode: 644 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/cadvisor/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/cadvisor/init.sls new file mode 100644 index 0000000..5ee3b32 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/cadvisor/init.sls @@ -0,0 +1,3 @@ +delete_cadvisor_manifest: + file.absent: + - name: /etc/kubernetes/manifests/cadvisor.manifest diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/cni/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/cni/init.sls new file mode 100644 index 0000000..544f33f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/cni/init.sls @@ -0,0 +1,20 @@ +/opt/cni: + file.directory: + - user: root + - group: root + - mode: 755 + - makedirs: True + +# These are all available CNI network plugins. +cni-tar: + archive: + - extracted + - user: root + - name: /opt/cni + - makedirs: True + - source: https://storage.googleapis.com/kubernetes-release/network-plugins/cni-09214926.tar.gz + - tar_options: v + - source_hash: md5=58f8631f912dd88be6a0920775db7274 + - archive_format: tar + - if_missing: /opt/cni/bin + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/debian-auto-upgrades/20auto-upgrades b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/debian-auto-upgrades/20auto-upgrades new file mode 100644 index 0000000..2bb25d7 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/debian-auto-upgrades/20auto-upgrades @@ -0,0 +1,4 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; + +APT::Periodic::AutocleanInterval "7"; diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/debian-auto-upgrades/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/debian-auto-upgrades/init.sls new file mode 100644 index 0000000..79e28a6 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/debian-auto-upgrades/init.sls @@ -0,0 +1,13 @@ +{% if grains['os_family'] == 'Debian' %} +unattended-upgrades: + pkg.installed + +'/etc/apt/apt.conf.d/20auto-upgrades': + file.managed: + - source: salt://debian-auto-upgrades/20auto-upgrades + - user: root + - group: root + - mode: 644 + - require: + - pkg: unattended-upgrades +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/default b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/default new file mode 100644 index 0000000..706f082 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/default @@ -0,0 +1,8 @@ +{% set docker_opts = "" -%} +{% if grains.docker_opts is defined and grains.docker_opts -%} + {% set docker_opts = grains.docker_opts -%} +{% endif -%} + +DOCKER_OPTS='{{docker_opts}}' +OPTIONS='{{docker_opts}}' +DOCKER_CERT_PATH=/etc/docker diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-defaults b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-defaults new file mode 100644 index 0000000..2eee8d7 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-defaults @@ -0,0 +1,15 @@ +{% set grains_opts = grains.get('docker_opts', '') -%} +{% set e2e_opts = '' -%} +{% if pillar.get('e2e_storage_test_environment', '').lower() == 'true' -%} +{% set e2e_opts = '-s devicemapper' -%} +{% endif -%} +{% set bridge_opts = "--bridge=cbr0" %} +{% if pillar.get('network_provider', '').lower() == 'kubenet' %} + {% set bridge_opts = "" %} +{% endif -%} +{% set log_level = "--log-level=warn" -%} +{% if pillar['docker_test_log_level'] is defined -%} + {% set log_level = pillar['docker_test_log_level'] -%} +{% endif -%} +DOCKER_OPTS="{{grains_opts}} {{e2e_opts}} {{bridge_opts}} --iptables=false --ip-masq=false {{log_level}}" +DOCKER_NOFILE=1000000 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck new file mode 100755 index 0000000..9167567 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is intended to be run periodically, to check the health +# of docker. If it detects a failure, it will restart docker using systemctl. + +if timeout 10 docker version > /dev/null; then + exit 0 +fi + +echo "docker failed" +echo "Giving docker 30 seconds grace before restarting" +sleep 30 + +if timeout 10 docker version > /dev/null; then + echo "docker recovered" + exit 0 +fi + +echo "docker still down; triggering docker restart" +systemctl restart docker + +echo "Waiting 60 seconds to give docker time to start" +sleep 60 + +if timeout 10 docker version > /dev/null; then + echo "docker recovered" + exit 0 +fi + +echo "docker still failing" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck.service b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck.service new file mode 100644 index 0000000..b4ecfaf --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck.service @@ -0,0 +1,9 @@ +[Unit] +Description=Run docker-healthcheck once + +[Service] +Type=oneshot +ExecStart=/opt/kubernetes/helpers/docker-healthcheck + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck.timer b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck.timer new file mode 100644 index 0000000..3d252e4 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-healthcheck.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Trigger docker-healthcheck periodically + +[Timer] +OnUnitInactiveSec=10s +Unit=docker-healthcheck.service + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-init b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-init new file mode 100755 index 0000000..18c7c5c --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-init @@ -0,0 +1,149 @@ +#!/bin/sh + +### BEGIN INIT INFO +# Provides: docker +# Required-Start: $syslog $remote_fs +# Required-Stop: $syslog $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Create lightweight, portable, self-sufficient containers. +# Description: +# Docker is an open-source project to easily create lightweight, portable, +# self-sufficient containers from any application. The same container that a +# developer builds and tests on a laptop can run at scale, in production, on +# VMs, bare metal, OpenStack clusters, public clouds and more. +### END INIT INFO + +export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin + +BASE=$(basename $0) + +# modify these in /etc/default/$BASE (/etc/default/docker) +# HACK: DOCKER overridable to allow image builder to run this, too. +DOCKER=${DOCKER:-/usr/bin/$BASE} +DOCKER_PIDFILE=/var/run/$BASE.pid +DOCKER_LOGFILE=/var/log/$BASE.log +DOCKER_OPTS= +DOCKER_DESC="Docker" + +# Get lsb functions +. /lib/lsb/init-functions + +if [ -f /etc/default/$BASE ]; then + . /etc/default/$BASE +fi + +# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it) +if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then + log_failure_msg "$DOCKER_DESC is managed via upstart, try using service $BASE $1" + exit 1 +fi + +# Check docker is present +if [ ! -x $DOCKER ]; then + log_failure_msg "$DOCKER not present or not executable" + exit 1 +fi + +fail_unless_root() { + if [ "$(id -u)" != '0' ]; then + log_failure_msg "$DOCKER_DESC must be run as root" + exit 1 + fi +} + +cgroupfs_mount() { + # see also https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount + if grep -v '^#' /etc/fstab | grep -q cgroup \ + || [ ! -e /proc/cgroups ] \ + || [ ! -d /sys/fs/cgroup ]; then + return + fi + if ! mountpoint -q /sys/fs/cgroup; then + mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup + fi + ( + cd /sys/fs/cgroup + for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do + mkdir -p $sys + if ! mountpoint -q $sys; then + if ! mount -n -t cgroup -o $sys cgroup $sys; then + rmdir $sys || true + fi + fi + done + ) +} + +case "$1" in + mount) # HACK: Used in image build + fail_unless_root + + cgroupfs_mount + + log_end_msg $? + ;; + + start) + fail_unless_root + + cgroupfs_mount + + # TODO: Find a better place to do this! + echo 1 > /sys/fs/cgroup/memory/memory.use_hierarchy + + touch "$DOCKER_LOGFILE" + chgrp docker "$DOCKER_LOGFILE" + + if [ -n "$DOCKER_NOFILE" ]; then + ulimit -n $DOCKER_NOFILE + fi + + if [ -n "$DOCKER_LOCKEDMEMORY" ]; then + ulimit -l $DOCKER_LOCKEDMEMORY + fi + + log_begin_msg "Starting $DOCKER_DESC: $BASE" + start-stop-daemon --start --background \ + --no-close \ + --exec "$DOCKER" \ + --pidfile "$DOCKER_PIDFILE" \ + -- \ + daemon -p "$DOCKER_PIDFILE" \ + $DOCKER_OPTS \ + >> "$DOCKER_LOGFILE" 2>&1 + log_end_msg $? + ;; + + stop) + fail_unless_root + log_begin_msg "Stopping $DOCKER_DESC: $BASE" + start-stop-daemon --stop --pidfile "$DOCKER_PIDFILE" + log_end_msg $? + ;; + + restart) + fail_unless_root + docker_pid=`cat "$DOCKER_PIDFILE" 2>/dev/null` + [ -n "$docker_pid" ] \ + && ps -p $docker_pid > /dev/null 2>&1 \ + && $0 stop + $0 start + ;; + + force-reload) + fail_unless_root + $0 restart + ;; + + status) + status_of_proc -p "$DOCKER_PIDFILE" "$DOCKER" docker + ;; + + *) + echo "Usage: $0 {start|stop|restart|status}" + exit 1 + ;; +esac + +exit 0 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-prestart b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-prestart new file mode 100755 index 0000000..ea23d6d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker-prestart @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is intended to be run before we start Docker. + +# cleanup docker network checkpoint to avoid running into known issue +# of docker (https://github.com/docker/docker/issues/18283) +rm -rf /var/lib/docker/network + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker.list b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker.list new file mode 100644 index 0000000..02fb6ad --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker.list @@ -0,0 +1 @@ +deb https://apt.dockerproject.org/repo debian-{{ salt['grains.get']('oscodename') }} main diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker.service b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker.service new file mode 100644 index 0000000..a5e02c9 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/docker.service @@ -0,0 +1,21 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +After=network.target docker.socket +Requires=docker.socket + +[Service] +Type=notify +EnvironmentFile={{ environment_file }} +ExecStart=/usr/bin/docker daemon -H fd:// "$DOCKER_OPTS" +MountFlags=slave +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +Restart=always +RestartSec=2s +StartLimitInterval=0 +ExecStartPre=/opt/kubernetes/helpers/docker-prestart + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/init.sls new file mode 100644 index 0000000..6e1f5a8 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/docker/init.sls @@ -0,0 +1,419 @@ +{% if pillar.get('is_systemd') %} + {% set environment_file = '/etc/sysconfig/docker' %} +{% else %} + {% set environment_file = '/etc/default/docker' %} +{% endif %} + +bridge-utils: + pkg.installed + +{% if grains.os_family == 'RedHat' %} + +{{ environment_file }}: + file.managed: + - source: salt://docker/default + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + +{% if (grains.os == 'Fedora' and grains.osrelease_info[0] >= 22) or (grains.os == 'CentOS' and grains.osrelease_info[0] >= 7) %} + +docker: + pkg: + - installed + service.running: + - enable: True + - require: + - pkg: docker + - watch: + - file: {{ environment_file }} + - pkg: docker + +{% else %} + +docker-io: + pkg: + - installed + +docker: + service.running: + - enable: True + - require: + - pkg: docker-io + - watch: + - file: {{ environment_file }} + - pkg: docker-io + +{% endif %} +{% elif grains.cloud is defined and grains.cloud == 'vsphere' and grains.os == 'Debian' and grains.osrelease_info[0] >=8 %} + +{% if pillar.get('is_systemd') %} + +/opt/kubernetes/helpers/docker-prestart: + file.managed: + - source: salt://docker/docker-prestart + - user: root + - group: root + - mode: 755 + +{{ pillar.get('systemd_system_path') }}/docker.service: + file.managed: + - source: salt://docker/docker.service + - template: jinja + - user: root + - group: root + - mode: 644 + - defaults: + environment_file: {{ environment_file }} + - require: + - file: /opt/kubernetes/helpers/docker-prestart + +# The docker service.running block below doesn't work reliably +# Instead we run our script which e.g. does a systemd daemon-reload +# But we keep the service block below, so it can be used by dependencies +# TODO: Fix this +fix-service-docker: + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce docker + - watch: + - file: {{ pillar.get('systemd_system_path') }}/docker.service + - file: {{ environment_file }} +{% endif %} + +{{ environment_file }}: + file.managed: + - source: salt://docker/docker-defaults + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - require: + - pkg: docker-engine + +'apt-key': + cmd.run: + - name: 'apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D' + - unless: 'apt-key finger | grep "5811 8E89"' + +'apt-update': + cmd.wait: + - name: '/usr/bin/apt-get update -y' + - require: + - cmd : 'apt-key' + +lxc-docker: + pkg: + - purged + +docker-io: + pkg: + - purged + +cbr0: + network.managed: + - enabled: True + - type: bridge + - proto: dhcp + - ports: none + - bridge: cbr0 + - delay: 0 + - bypassfirewall: True + - require_in: + - service: docker + +/etc/apt/sources.list.d/docker.list: + file.managed: + - source: salt://docker/docker.list + - template: jinja + - user: root + - group: root + - mode: 644 + - require: + - cmd: 'apt-update' + +# restricting docker version to 1.9. with older version of docker we are facing +# issue https://github.com/docker/docker/issues/18793. +# newer version of docker 1.10.0 is not well tested yet. +# full comments: https://github.com/kubernetes/kubernetes/pull/20851 +docker-engine: + pkg: + - installed + - version: 1.9.* + - require: + - file: /etc/apt/sources.list.d/docker.list +docker: + service.running: + - enable: True + - require: + - file: {{ environment_file }} + - watch: + - file: {{ environment_file }} + +{% else %} + +{% if grains.cloud is defined + and grains.cloud == 'gce' %} +# The default GCE images have ip_forwarding explicitly set to 0. +# Here we take care of commenting that out. +/etc/sysctl.d/11-gce-network-security.conf: + file.replace: + - pattern: '^net.ipv4.ip_forward=0' + - repl: '# net.ipv4.ip_forward=0' + +/etc/init.d/docker: + file.managed: + - source: salt://docker/docker-init + - template: jinja + - user: root + - group: root + - mode: 755 + - makedirs: true +{% endif %} + +# Work around Salt #18089: https://github.com/saltstack/salt/issues/18089 +/etc/sysctl.d/99-salt.conf: + file.touch + +# TODO: This should really be based on network strategy instead of os_family +net.ipv4.ip_forward: + sysctl.present: + - value: 1 + +{{ environment_file }}: + file.managed: + - source: salt://docker/docker-defaults + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + +# Docker is on the ContainerVM image by default. The following +# variables are provided for other cloud providers, and for testing and dire circumstances, to allow +# overriding the Docker version that's in a ContainerVM image. +# +# To change: +# +# 1. Find new deb name at: +# http://apt.dockerproject.org/repo/pool/main/d/docker-engine +# 2. Download based on that: +# curl -O http://apt.dockerproject.org/repo/pool/main/d/docker-engine/ +# 3. Upload to GCS: +# gsutil cp gs://kubernetes-release/docker/ +# 4. Make it world readable: +# gsutil acl ch -R -g all:R gs://kubernetes-release/docker/ +# 5. Get a hash of the deb: +# shasum +# 6. Update override_deb, override_deb_sha1, override_docker_ver with new +# deb name, new hash and new version + +{% set storage_base='https://storage.googleapis.com/kubernetes-release/docker/' %} + +{% set override_deb_url='' %} + +{% if grains.get('cloud', '') == 'gce' + and grains.get('os_family', '') == 'Debian' + and grains.get('oscodename', '') == 'wheezy' -%} +{% set docker_pkg_name='' %} +{% set override_deb='' %} +{% set override_deb_sha1='' %} +{% set override_docker_ver='' %} + +{% elif grains.get('cloud', '') == 'aws' + and grains.get('os_family', '') == 'Debian' + and grains.get('oscodename', '') == 'jessie' -%} +# TODO: Get from google storage? +{% set docker_pkg_name='docker-engine' %} +{% set override_docker_ver='1.9.1-0~jessie' %} +{% set override_deb='docker-engine_1.9.1-0~jessie_amd64.deb' %} +{% set override_deb_url='http://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.9.1-0~jessie_amd64.deb' %} +{% set override_deb_sha1='c58c39008fd6399177f6b2491222e4438f518d78' %} + +# Ubuntu presents as os_family=Debian, osfullname=Ubuntu +{% elif grains.get('cloud', '') == 'aws' + and grains.get('os_family', '') == 'Debian' + and grains.get('oscodename', '') == 'trusty' -%} +# TODO: Get from google storage? +{% set docker_pkg_name='docker-engine' %} +{% set override_docker_ver='1.9.1-0~trusty' %} +{% set override_deb='docker-engine_1.9.1-0~trusty_amd64.deb' %} +{% set override_deb_url='http://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.9.1-0~trusty_amd64.deb' %} +{% set override_deb_sha1='ce728172ab29f9fdacfffffe2e2f88a144f23875' %} + +{% elif grains.get('cloud', '') == 'aws' + and grains.get('os_family', '') == 'Debian' + and grains.get('oscodename', '') == 'vivid' -%} +# TODO: Get from google storage? +{% set docker_pkg_name='docker-engine' %} +{% set override_docker_ver='1.9.1-0~vivid' %} +{% set override_deb='docker-engine_1.9.1-0~vivid_amd64.deb' %} +{% set override_deb_url='http://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.9.1-0~vivid_amd64.deb' %} +{% set override_deb_sha1='81741f6f16630632de53762c5554238d57b3b9cb' %} + +{% elif grains.get('cloud', '') == 'aws' + and grains.get('os_family', '') == 'Debian' + and grains.get('oscodename', '') == 'wily' -%} +# TODO: Get from google storage? +{% set docker_pkg_name='docker-engine' %} +{% set override_docker_ver='1.9.1-0~wily' %} +{% set override_deb='docker-engine_1.9.1-0~wily_amd64.deb' %} +{% set override_deb_url='http://apt.dockerproject.org/repo/pool/main/d/docker-engine/docker-engine_1.9.1-0~wily_amd64.deb' %} +{% set override_deb_sha1='a505fd49372cf836f5b9ed953053c50b3381dbfd' %} + +{% else %} +{% set docker_pkg_name='lxc-docker-1.7.1' %} +{% set override_docker_ver='1.7.1' %} +{% set override_deb='lxc-docker-1.7.1_1.7.1_amd64.deb' %} +{% set override_deb_sha1='81abef31dd2c616883a61f85bfb294d743b1c889' %} +{% endif %} + +{% if override_deb_url == '' %} +{% set override_deb_url=storage_base + override_deb %} +{% endif %} + +{% if override_docker_ver != '' %} +purge-old-docker-package: + pkg.removed: + - pkgs: + - lxc-docker-1.6.2 + +/var/cache/docker-install/{{ override_deb }}: + file.managed: + - source: {{ override_deb_url }} + - source_hash: sha1={{ override_deb_sha1 }} + - user: root + - group: root + - mode: 644 + - makedirs: true + +# Drop the license file into /usr/share so that everything is crystal clear. +/usr/share/doc/docker/apache.txt: + file.managed: + - source: {{ storage_base }}apache2.txt + - source_hash: sha1=2b8b815229aa8a61e483fb4ba0588b8b6c491890 + - user: root + - group: root + - mode: 644 + - makedirs: true + +docker-upgrade: + cmd.run: + - name: /opt/kubernetes/helpers/pkg install-no-start {{ docker_pkg_name }} {{ override_docker_ver }} /var/cache/docker-install/{{ override_deb }} + - require: + - file: /var/cache/docker-install/{{ override_deb }} +{% endif %} # end override_docker_ver != '' + +{% if pillar.get('is_systemd') %} + +/opt/kubernetes/helpers/docker-prestart: + file.managed: + - source: salt://docker/docker-prestart + - user: root + - group: root + - mode: 755 + +# Default docker systemd unit file doesn't use an EnvironmentFile; replace it with one that does. +{{ pillar.get('systemd_system_path') }}/docker.service: + file.managed: + - source: salt://docker/docker.service + - template: jinja + - user: root + - group: root + - mode: 644 + - defaults: + environment_file: {{ environment_file }} + - require: + - file: /opt/kubernetes/helpers/docker-prestart + +# The docker service.running block below doesn't work reliably +# Instead we run our script which e.g. does a systemd daemon-reload +# But we keep the service block below, so it can be used by dependencies +# TODO: Fix this +fix-service-docker: + cmd.wait: + - name: /opt/kubernetes/helpers/services enable docker + - watch: + - file: {{ pillar.get('systemd_system_path') }}/docker.service + - file: {{ environment_file }} +{% if override_docker_ver != '' %} + - require: + - cmd: docker-upgrade +{% endif %} + +/opt/kubernetes/helpers/docker-healthcheck: + file.managed: + - source: salt://docker/docker-healthcheck + - user: root + - group: root + - mode: 755 + +{{ pillar.get('systemd_system_path') }}/docker-healthcheck.service: + file.managed: + - source: salt://docker/docker-healthcheck.service + - template: jinja + - user: root + - group: root + - mode: 644 + +{{ pillar.get('systemd_system_path') }}/docker-healthcheck.timer: + file.managed: + - source: salt://docker/docker-healthcheck.timer + - template: jinja + - user: root + - group: root + - mode: 644 + +# Tell systemd to load the timer +fix-systemd-docker-healthcheck-timer: + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce docker-healthcheck.timer + - watch: + - file: {{ pillar.get('systemd_system_path') }}/docker-healthcheck.timer + +# Trigger a first run of docker-healthcheck; needed because the timer fires 10s after the previous run. +fix-systemd-docker-healthcheck-service: + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce docker-healthcheck.service + - watch: + - file: {{ pillar.get('systemd_system_path') }}/docker-healthcheck.service + - require: + - cmd: fix-service-docker + +{% endif %} + +docker: +# Starting Docker is racy on aws for some reason. To be honest, since Monit +# is managing Docker restart we should probably just delete this whole thing +# but the kubernetes components use salt 'require' to set up a dag, and that +# complicated and scary to unwind. +# On AWS, we use a trick now... we don't start the docker service through Salt. +# Kubelet or our health checker will start it. But we use service.enabled, +# so we still have a `service: docker` node for our DAG. +{% if grains.cloud is defined and grains.cloud == 'aws' %} + service.enabled: +{% else %} + service.running: + - enable: True +{% endif %} +# If we put a watch on this, salt will try to start the service. +# We put the watch on the fixer instead +{% if not pillar.get('is_systemd') %} + - watch: + - file: {{ environment_file }} +{% if override_docker_ver != '' %} + - cmd: docker-upgrade +{% endif %} +{% endif %} + - require: + - file: {{ environment_file }} +{% if override_docker_ver != '' %} + - cmd: docker-upgrade +{% endif %} +{% if pillar.get('is_systemd') %} + - cmd: fix-service-docker +{% endif %} +{% endif %} # end grains.os_family != 'RedHat' + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/e2e/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/e2e/init.sls new file mode 100644 index 0000000..8332649 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/e2e/init.sls @@ -0,0 +1,28 @@ +e2e: + # Install various packages required by e2e tests to all hosts. + pkg.installed: + - refresh: true + - pkgs: + - targetcli + - ceph +{% if grains['os_family'] == 'RedHat' %} + - glusterfs-fuse + - rbd-fuse + - iscsi-initiator-utils + - nfs-utils +{% else %} + - glusterfs-client + - open-iscsi + - iscsitarget-dkms + - nfs-common +{% endif %} + + + +{% if grains['os_family'] == 'Debian' %} +# On Debian, re-start open-iscsi to generate unique +# /etc/iscsi/initiatorname.iscsi +open-iscsi: + cmd.run: + - name: 'service open-iscsi restart' +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/etcd/etcd.manifest b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/etcd/etcd.manifest new file mode 100644 index 0000000..4bf4851 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/etcd/etcd.manifest @@ -0,0 +1,65 @@ +{ +"apiVersion": "v1", +"kind": "Pod", +"metadata": { + "name":"etcd-server{{ suffix }}", + "namespace": "kube-system" +}, +"spec":{ +"hostNetwork": true, +"containers":[ + { + "name": "etcd-container", + "image": "gcr.io/google_containers/etcd:2.2.1", + "resources": { + "requests": { + "cpu": {{ cpulimit }} + } + }, + "command": [ + "/bin/sh", + "-c", + "/usr/local/bin/etcd --listen-peer-urls http://127.0.0.1:{{ server_port }} --addr 127.0.0.1:{{ port }} --bind-addr 127.0.0.1:{{ port }} --data-dir /var/etcd/data{{ suffix }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" + ], + "livenessProbe": { + "httpGet": { + "host": "127.0.0.1", + "port": {{ port }}, + "path": "/health" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 15 + }, + "ports":[ + { "name": "serverport", + "containerPort": {{ server_port }}, + "hostPort": {{ server_port }} + },{ + "name": "clientport", + "containerPort": {{ port }}, + "hostPort": {{ port }} + } + ], + "volumeMounts": [ + {"name": "varetcd", + "mountPath": "/var/etcd", + "readOnly": false + }, + {"name": "varlogetcd", + "mountPath": "/var/log/etcd{{ suffix }}.log", + "readOnly": false + } + ] + } +], +"volumes":[ + { "name": "varetcd", + "hostPath": { + "path": "/mnt/master-pd/var/etcd"} + }, + { "name": "varlogetcd", + "hostPath": { + "path": "/var/log/etcd{{ suffix }}.log"} + } +] +}} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/etcd/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/etcd/init.sls new file mode 100644 index 0000000..edd9f65 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/etcd/init.sls @@ -0,0 +1,83 @@ +# Early configurations of Kubernetes ran etcd on the host and as part of a migration step, we began to delete the host etcd +# It's possible though that the host has configured a separate etcd to configure other services like Flannel +# In that case, we do not want Salt to remove or stop the host service +# Note: its imperative that the host installed etcd not conflict with the Kubernetes managed etcd +{% if grains['keep_host_etcd'] is not defined %} + +delete_etc_etcd_dir: + file.absent: + - name: /etc/etcd + +delete_etcd_conf: + file.absent: + - name: /etc/etcd/etcd.conf + +delete_etcd_default: + file.absent: + - name: /etc/default/etcd + +{% if pillar.get('is_systemd') %} +delete_etcd_service_file: + file.absent: + - name: {{ pillar.get('systemd_system_path') }}/etcd.service +{% endif %} + +delete_etcd_initd: + file.absent: + - name: /etc/init.d/etcd + +#stop legacy etcd_service +stop_etcd-service: + service.dead: + - name: etcd + - enable: None + +{% endif %} + +touch /var/log/etcd.log: + cmd.run: + - creates: /var/log/etcd.log + +touch /var/log/etcd-events.log: + cmd.run: + - creates: /var/log/etcd-events.log + +/var/etcd: + file.directory: + - user: root + - group: root + - dir_mode: 700 + - recurse: + - user + - group + - mode + +/etc/kubernetes/manifests/etcd.manifest: + file.managed: + - source: salt://etcd/etcd.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - context: + suffix: "" + port: 4001 + server_port: 2380 + cpulimit: '"200m"' + +/etc/kubernetes/manifests/etcd-events.manifest: + file.managed: + - source: salt://etcd/etcd.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - context: + suffix: "-events" + port: 4002 + server_port: 2381 + cpulimit: '"100m"' diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/flannel-server.manifest b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/flannel-server.manifest new file mode 100644 index 0000000..c679f21 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/flannel-server.manifest @@ -0,0 +1,110 @@ +{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": { + "name": "flannel-server", + "namespace": "kube-system", + "labels": { + "app": "flannel-server", + "version": "v0.1" + } + }, + "spec": { + "volumes": [ + { + "name": "varlog", + "hostPath": { + "path": "/var/log" + } + }, + { + "name": "etcdstorage", + "emptyDir": {} + }, + { + "name": "networkconfig", + "hostPath": { + "path": "/etc/kubernetes/network.json" + } + } + ], + "containers": [ + { + "name": "flannel-server-helper", + "image": "gcr.io/google_containers/flannel-server-helper:0.1", + "args": [ + "--network-config=/etc/kubernetes/network.json", + "--etcd-prefix=/kubernetes.io/network", + "--etcd-server=http://127.0.0.1:{{ etcd_port }}" + ], + "volumeMounts": [ + { + "name": "networkconfig", + "mountPath": "/etc/kubernetes/network.json" + } + ], + "imagePullPolicy": "Always" + }, + { + "name": "flannel-container", + "image": "quay.io/coreos/flannel:0.5.5", + "command": [ + "/bin/sh", + "-c", + "/opt/bin/flanneld -listen 0.0.0.0:10253 -etcd-endpoints http://127.0.0.1:{{ etcd_port }} -etcd-prefix /kubernetes.io/network 1>>/var/log/flannel_server.log 2>&1" + ], + "ports": [ + { + "hostPort": 10253, + "containerPort": 10253 + } + ], + "resources": { + "requests": { + "cpu": {{ cpulimit }} + } + }, + "volumeMounts": [ + { + "name": "varlog", + "mountPath": "/var/log" + } + ] + }, + { + "name": "etcd-container", + "image": "gcr.io/google_containers/etcd:2.2.1", + "command": [ + "/bin/sh", + "-c", + "/usr/local/bin/etcd --listen-peer-urls http://127.0.0.1:{{ etcd_peer_port }} --addr 127.0.0.1:{{ etcd_port }} --bind-addr 127.0.0.1:{{ etcd_port }} --data-dir /var/etcd/data 1>>/var/log/etcd_flannel.log 2>&1" + ], + "livenessProbe": { + "httpGet": { + "host": "127.0.0.1", + "port": {{ etcd_port }}, + "path": "/health" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 15 + }, + "resources": { + "requests": { + "cpu": {{ cpulimit }} + } + }, + "volumeMounts": [ + { + "name": "varlog", + "mountPath": "/var/log" + }, + { + "name": "etcdstorage", + "mountPath": "/var/etcd/data" + } + ] + } + ], + "hostNetwork": true + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/init.sls new file mode 100644 index 0000000..5712fea --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/init.sls @@ -0,0 +1,29 @@ +touch /var/log/flannel.log: + cmd.run: + - creates: /var/log/flannel.log + +touch /var/log/etcd_flannel.log: + cmd.run: + - creates: /var/log/etcd_flannel.log + +/etc/kubernetes/network.json: + file.managed: + - source: salt://flannel-server/network.json + - makedirs: True + - user: root + - group: root + - mode: 755 + +/etc/kubernetes/manifests/flannel-server.manifest: + file.managed: + - source: salt://flannel-server/flannel-server.manifest + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - template: jinja + - context: + etcd_port: 4003 + etcd_peer_port: 2382 + cpulimit: '"100m"' diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/network.json b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/network.json new file mode 100644 index 0000000..b0a6bd4 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel-server/network.json @@ -0,0 +1,8 @@ +{ + "Network": "172.16.0.0/12", + "SubnetLen": 24, + "Backend": { + "Type": "vxlan", + "VNI": 1 + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/default b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/default new file mode 100644 index 0000000..a494006 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/default @@ -0,0 +1,6 @@ +{% if grains.api_servers is defined -%} + {% set daemon_args = "-remote " + grains.api_servers + ":10253" -%} +{% else -%} + {% set daemon_args = "-remote 127.0.0.1:10253" -%} +{% endif -%} +DAEMON_ARGS="{{daemon_args}}" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/init.sls new file mode 100644 index 0000000..25ba9c6 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/init.sls @@ -0,0 +1,44 @@ +# TODO: Run flannel daemon in a static pod once we've moved the overlay network +# setup into a network plugin. +flannel-tar: + archive: + - extracted + - user: root + - name: /usr/local/src + - makedirs: True + - source: https://storage.googleapis.com/kubernetes-release/flannel/flannel-0.5.5-linux-amd64.tar.gz + - tar_options: v + - source_hash: md5=972c717254775bef528f040af804f2cc + - archive_format: tar + - if_missing: /usr/local/src/flannel/flannel-0.5.5/ + +flannel-symlink: + file.symlink: + - name: /usr/local/bin/flanneld + - target: /usr/local/src/flannel-0.5.5/flanneld + - force: true + - watch: + - archive: flannel-tar + +/etc/default/flannel: + file.managed: + - source: salt://flannel/default + - template: jinja + - user: root + - group: root + - mode: 644 + +/etc/init.d/flannel: + file.managed: + - source: salt://flannel/initd + - user: root + - group: root + - mode: 755 + +flannel: + service.running: + - enable: True + - watch: + - file: /usr/local/bin/flanneld + - file: /etc/init.d/flannel + - file: /etc/default/flannel diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/initd b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/initd new file mode 100644 index 0000000..c8e1d3b --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/flannel/initd @@ -0,0 +1,126 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: flanneld +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Flannel daemon +# Description: +# Flannel daemon. +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Flannel overlay network daemon" +NAME=flannel +DAEMON=/usr/local/bin/flanneld +DAEMON_ARGS="--ip-masq" +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=root + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Avoid a potential race at boot time when both monit and init.d start + # the same service + PIDS=$(pidof $DAEMON) + for PID in ${PIDS}; do + kill -9 $PID + done + + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --background --no-close \ + --make-pidfile --pidfile $PIDFILE \ + --exec $DAEMON -c $DAEMON_USER --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --background --no-close \ + --make-pidfile --pidfile $PIDFILE \ + --exec $DAEMON -c $DAEMON_USER -- \ + $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-es/fluentd-es.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-es/fluentd-es.yaml new file mode 100644 index 0000000..aea6812 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-es/fluentd-es.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + name: fluentd-elasticsearch + namespace: kube-system + labels: + k8s-app: fluentd-logging +spec: + containers: + - name: fluentd-elasticsearch + image: gcr.io/google_containers/fluentd-elasticsearch:1.15 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 30 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-es/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-es/init.sls new file mode 100644 index 0000000..0c8c41a --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-es/init.sls @@ -0,0 +1,10 @@ +{% if grains['roles'][0] != 'kubernetes-master' -%} +/etc/kubernetes/manifests/fluentd-es.yaml: + file.managed: + - source: salt://fluentd-es/fluentd-es.yaml + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml new file mode 100644 index 0000000..44cca43 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Pod +metadata: + name: fluentd-cloud-logging + namespace: kube-system + labels: + k8s-app: fluentd-logging +spec: + dnsPolicy: Default + containers: + - name: fluentd-cloud-logging + image: gcr.io/google_containers/fluentd-gcp:1.18 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + env: + - name: FLUENTD_ARGS + value: -q + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 30 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-gcp/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-gcp/init.sls new file mode 100644 index 0000000..5d38e2a --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/fluentd-gcp/init.sls @@ -0,0 +1,8 @@ +/etc/kubernetes/manifests/fluentd-gcp.yaml: + file.managed: + - source: salt://fluentd-gcp/fluentd-gcp.yaml + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/init.sls new file mode 100644 index 0000000..5d66fa2 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/init.sls @@ -0,0 +1,45 @@ +{% set master_extra_sans=grains.get('master_extra_sans', '') %} +{% if grains.cloud is defined %} + {% if grains.cloud == 'gce' %} + {% set cert_ip='_use_gce_external_ip_' %} + {% endif %} + {% if grains.cloud == 'aws' %} + {% set cert_ip='_use_aws_external_ip_' %} + {% endif %} + {% if grains.cloud == 'vsphere' %} + {% set cert_ip=grains.ip_interfaces.eth0[0] %} + {% endif %} +{% endif %} + +# If there is a pillar defined, override any defaults. +{% if pillar['cert_ip'] is defined %} + {% set cert_ip=pillar['cert_ip'] %} +{% endif %} + +{% set certgen="make-cert.sh" %} +{% if cert_ip is defined %} + {% set certgen="make-ca-cert.sh" %} +{% endif %} + +openssl: + pkg.installed: [] + +kube-cert: + group.present: + - system: True + +kubernetes-cert: + cmd.script: + - unless: test -f /srv/kubernetes/server.cert + - source: salt://generate-cert/{{certgen}} +{% if cert_ip is defined %} + - args: {{cert_ip}} {{master_extra_sans}} + - require: + - pkg: curl +{% endif %} + - cwd: / + - user: root + - group: root + - shell: /bin/bash + - require: + - pkg: openssl diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/make-ca-cert.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/make-ca-cert.sh new file mode 100755 index 0000000..575c9ed --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/make-ca-cert.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +DEBUG="${DEBUG:-false}" + +if [ "${DEBUG}" == "true" ]; then + set -x +fi + +cert_ip=$1 +extra_sans=${2:-} +cert_dir=${CERT_DIR:-/srv/kubernetes} +cert_group=${CERT_GROUP:-kube-cert} + +mkdir -p "$cert_dir" + +use_cn=false + +# TODO: Add support for discovery on other providers? +if [ "$cert_ip" == "_use_gce_external_ip_" ]; then + cert_ip=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip) +fi + +if [ "$cert_ip" == "_use_aws_external_ip_" ]; then + # If there's no public IP assigned (e.g. this host is running on an internal subnet in a VPC), then + # curl will happily spit out the contents of AWS's 404 page and an exit code of zero. + # + # The string containing the 404 page trips up one of easyrsa's calls to openssl later; whichever + # one creates the CA certificate, because the 404 page is > 64 characters. + if cert_ip=$(curl -f -s http://169.254.169.254/latest/meta-data/public-ipv4); then + : + else + cert_ip=$(curl -f -s http://169.254.169.254/latest/meta-data/local-ipv4) + fi +fi + +sans="IP:${cert_ip}" +if [[ -n "${extra_sans}" ]]; then + sans="${sans},${extra_sans}" +fi + +tmpdir=$(mktemp -d -t kubernetes_cacert.XXXXXX) +trap 'rm -rf "${tmpdir}"' EXIT +cd "${tmpdir}" + +# TODO: For now, this is a patched tool that makes subject-alt-name work, when +# the fix is upstream move back to the upstream easyrsa. This is cached in GCS +# but is originally taken from: +# https://github.com/brendandburns/easy-rsa/archive/master.tar.gz +# +# To update, do the following: +# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz +# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz +# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz +# +# Due to GCS caching of public objects, it may take time for this to be widely +# distributed. +# +# Use ~/kube/easy-rsa.tar.gz if it exists, so that it can be +# pre-pushed in cases where an outgoing connection is not allowed. +if [ -f ~/kube/easy-rsa.tar.gz ]; then + ln -s ~/kube/easy-rsa.tar.gz . +else + curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1 +fi +tar xzf easy-rsa.tar.gz > /dev/null 2>&1 + +cd easy-rsa-master/easyrsa3 +./easyrsa init-pki > /dev/null 2>&1 +./easyrsa --batch "--req-cn=$cert_ip@`date +%s`" build-ca nopass > /dev/null 2>&1 +if [ $use_cn = "true" ]; then + ./easyrsa build-server-full $cert_ip nopass > /dev/null 2>&1 + cp -p pki/issued/$cert_ip.crt "${cert_dir}/server.cert" > /dev/null 2>&1 + cp -p pki/private/$cert_ip.key "${cert_dir}/server.key" > /dev/null 2>&1 +else + ./easyrsa --subject-alt-name="${sans}" build-server-full kubernetes-master nopass > /dev/null 2>&1 + cp -p pki/issued/kubernetes-master.crt "${cert_dir}/server.cert" > /dev/null 2>&1 + cp -p pki/private/kubernetes-master.key "${cert_dir}/server.key" > /dev/null 2>&1 +fi +./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1 +cp -p pki/ca.crt "${cert_dir}/ca.crt" +cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt" +cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key" +# Make server certs accessible to apiserver. +chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt" +chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/make-cert.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/make-cert.sh new file mode 100755 index 0000000..914ed1f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/generate-cert/make-cert.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cert_dir=${CERT_DIR:-/srv/kubernetes} +cert_group=${CERT_GROUP:-kube-cert} + +mkdir -p "$cert_dir" + +openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \ + -subj "/CN=kubernetes.invalid/O=Kubernetes" \ + -keyout "${cert_dir}/server.key" -out "${cert_dir}/server.cert" +chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert" +chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/helpers/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/helpers/init.sls new file mode 100644 index 0000000..7e5ad43 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/helpers/init.sls @@ -0,0 +1,14 @@ +{% if grains['cloud'] is defined and grains['cloud'] == 'aws' %} +/usr/share/google: + file.directory: + - user: root + - group: root + - dir_mode: 755 + +/usr/share/google/safe_format_and_mount: + file.managed: + - source: salt://helpers/safe_format_and_mount + - user: root + - group: root + - mode: 755 +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/helpers/safe_format_and_mount b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/helpers/safe_format_and_mount new file mode 100644 index 0000000..90be230 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/helpers/safe_format_and_mount @@ -0,0 +1,147 @@ +#! /bin/bash +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Mount a disk, formatting it if necessary. If the disk looks like it may +# have been formatted before, we will not format it. +# +# This script uses blkid and file to search for magic "formatted" bytes +# at the beginning of the disk. Furthermore, it attempts to use fsck to +# repair the filesystem before formatting it. + +FSCK=fsck.ext4 +MOUNT_OPTIONS="discard,defaults" +MKFS="mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 -F" +if [ -e /etc/redhat-release ]; then + if grep -q '6\..' /etc/redhat-release; then + # lazy_journal_init is not recognized in redhat 6 + MKFS="mkfs.ext4 -E lazy_itable_init=0 -F" + elif grep -q '7\..' /etc/redhat-release; then + FSCK=fsck.xfs + MKFS=mkfs.xfs + fi +fi + +LOGTAG=safe_format_and_mount +LOGFACILITY=user + +function log() { + local readonly severity=$1; shift; + logger -t ${LOGTAG} -p ${LOGFACILITY}.${severity} -s "$@" +} + +function log_command() { + local readonly log_file=$(mktemp) + local readonly retcode + log info "Running: $*" + $* > ${log_file} 2>&1 + retcode=$? + # only return the last 1000 lines of the logfile, just in case it's HUGE. + tail -1000 ${log_file} | logger -t ${LOGTAG} -p ${LOGFACILITY}.info -s + rm -f ${log_file} + return ${retcode} +} + +function help() { + cat >&2 < +EOF + exit 0 +} + +while getopts ":hf:o:m:" opt; do + case $opt in + h) help;; + f) FSCK=$OPTARG;; + o) MOUNT_OPTIONS=$OPTARG;; + m) MKFS=$OPTARG;; + -) break;; + \?) log error "Invalid option: -${OPTARG}"; exit 1;; + :) log "Option -${OPTARG} requires an argument."; exit 1;; + esac +done + +shift $(($OPTIND - 1)) +readonly DISK=$1 +readonly MOUNTPOINT=$2 + +[[ -z ${DISK} ]] && help +[[ -z ${MOUNTPOINT} ]] && help + +function disk_looks_unformatted() { + blkid ${DISK} + if [[ $? == 0 ]]; then + return 0 + fi + + local readonly file_type=$(file --special-files ${DISK}) + case ${file_type} in + *filesystem*) + return 0;; + esac + + return 1 +} + +function format_disk() { + log_command ${MKFS} ${DISK} +} + +function try_repair_disk() { + log_command ${FSCK} -a ${DISK} + local readonly fsck_return=$? + if [[ ${fsck_return} -ge 8 ]]; then + log error "Fsck could not correct errors on ${DISK}" + return 1 + fi + if [[ ${fsck_return} -gt 0 ]]; then + log warning "Fsck corrected errors on ${DISK}" + fi + return 0 +} + +function try_mount() { + local mount_retcode + try_repair_disk + + log_command mount -o ${MOUNT_OPTIONS} ${DISK} ${MOUNTPOINT} + mount_retcode=$? + if [[ ${mount_retcode} == 0 ]]; then + return 0 + fi + + # Check to see if it looks like a filesystem before formatting it. + disk_looks_unformatted ${DISK} + if [[ $? == 0 ]]; then + log error "Disk ${DISK} looks formatted but won't mount. Giving up." + return ${mount_retcode} + fi + + # The disk looks like it's not been formatted before. + format_disk + if [[ $? != 0 ]]; then + log error "Format of ${DISK} failed." + fi + + log_command mount -o ${MOUNT_OPTIONS} ${DISK} ${MOUNTPOINT} + mount_retcode=$? + if [[ ${mount_retcode} == 0 ]]; then + return 0 + fi + log error "Tried everything we could, but could not mount ${DISK}." + return ${mount_retcode} +} + +try_mount +exit $? diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-loadbalancing/glbc/default-svc.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-loadbalancing/glbc/default-svc.yaml new file mode 100644 index 0000000..1bd5932 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-loadbalancing/glbc/default-svc.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + # This must match the --default-backend-service argument of the l7 lb + # controller and is required because GCE mandates a default backend. + name: default-http-backend + namespace: kube-system + labels: + k8s-app: glbc + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "GLBCDefaultBackend" +spec: + # The default backend must be of type NodePort. + type: NodePort + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + k8s-app: glbc diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-loadbalancing/glbc/glbc-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-loadbalancing/glbc/glbc-controller.yaml new file mode 100644 index 0000000..406458d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-loadbalancing/glbc/glbc-controller.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: l7-lb-controller-v0.6.0 + namespace: kube-system + labels: + k8s-app: glbc + version: v0.6.0 + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "GLBC" +spec: + # There should never be more than 1 controller alive simultaneously. + replicas: 1 + selector: + k8s-app: glbc + version: v0.6.0 + template: + metadata: + labels: + k8s-app: glbc + version: v0.6.0 + name: glbc + kubernetes.io/cluster-service: "true" + spec: + terminationGracePeriodSeconds: 600 + containers: + - name: default-http-backend + # Any image is permissible as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.0 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + - image: gcr.io/google_containers/glbc:0.6.0 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + scheme: HTTP + initialDelaySeconds: 30 + # healthz reaches out to GCE + periodSeconds: 30 + timeoutSeconds: 5 + name: l7-lb-controller + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 50Mi + args: + - --default-backend-service=kube-system/default-http-backend + - --sync-period=300s diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/google/heapster-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/google/heapster-controller.yaml new file mode 100644 index 0000000..1370899 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/google/heapster-controller.yaml @@ -0,0 +1,68 @@ +{% set metrics_memory = "200Mi" -%} +{% set eventer_memory = "200Mi" -%} +{% set num_nodes = pillar.get('num_nodes', -1) -%} +{% if num_nodes >= 0 -%} + {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%} + {% set eventer_memory = (200 * 1024 + num_nodes * 500)|string + "Ki" -%} +{% endif -%} + +apiVersion: v1 +kind: ReplicationController +metadata: + name: heapster-v1.0.0 + namespace: kube-system + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: heapster + template: + metadata: + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" + spec: + containers: + - image: gcr.io/google_containers/heapster:v1.0.0 + name: heapster + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: {{ metrics_memory }} + requests: + cpu: 100m + memory: {{ metrics_memory }} + command: + - /heapster + - --source=kubernetes.summary_api:'' + - --sink=gcm + - --metric_resolution=60s + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs + readOnly: true + - image: gcr.io/google_containers/heapster:v1.0.0 + name: eventer + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: {{ eventer_memory }} + requests: + cpu: 100m + memory: {{ eventer_memory }} + command: + - /eventer + - --source=kubernetes:'' + - --sink=gcl + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs + readOnly: true + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/google/heapster-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/google/heapster-service.yaml new file mode 100644 index 0000000..31e8b96 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/google/heapster-service.yaml @@ -0,0 +1,14 @@ +kind: Service +apiVersion: v1 +metadata: + name: heapster + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Heapster" +spec: + ports: + - port: 80 + targetPort: 8082 + selector: + k8s-app: heapster diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml new file mode 100644 index 0000000..928c824 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -0,0 +1,69 @@ +{% set metrics_memory = "200Mi" -%} +{% set eventer_memory = "200Mi" -%} +{% set num_nodes = pillar.get('num_nodes', -1) -%} +{% if num_nodes >= 0 -%} + {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%} + {% set eventer_memory = (200 * 1024 + num_nodes * 500)|string + "Ki" -%} +{% endif -%} + +apiVersion: v1 +kind: ReplicationController +metadata: + name: heapster-v1.0.0 + namespace: kube-system + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: heapster + template: + metadata: + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" + spec: + containers: + - image: gcr.io/google_containers/heapster:v1.0.0 + name: heapster + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: {{ metrics_memory }} + requests: + cpu: 100m + memory: {{ metrics_memory }} + command: + - /heapster + - --source=kubernetes.summary_api:'' + - --sink=influxdb:http://monitoring-influxdb:8086 + - --sink=gcm:?metrics=autoscaling + - --metric_resolution=60s + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs + readOnly: true + - image: gcr.io/google_containers/heapster:v1.0.0 + name: eventer + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: {{ eventer_memory }} + requests: + cpu: 100m + memory: {{ eventer_memory }} + command: + - /eventer + - --source=kubernetes:'' + - --sink=gcl + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs + readOnly: true + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/grafana-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/grafana-service.yaml new file mode 100644 index 0000000..9140e8b --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/grafana-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: monitoring-grafana + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Grafana" +spec: + # On production clusters, consider setting up auth for grafana, and + # exposing Grafana either using a LoadBalancer or a public IP. + # type: LoadBalancer + ports: + - port: 80 + targetPort: 3000 + selector: + k8s-app: influxGrafana + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/heapster-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/heapster-controller.yaml new file mode 100644 index 0000000..de792b4 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -0,0 +1,56 @@ +{% set metrics_memory = "200Mi" -%} +{% set eventer_memory = "200Mi" -%} +{% set num_nodes = pillar.get('num_nodes', -1) -%} +{% if num_nodes >= 0 -%} + {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%} + {% set eventer_memory = (200 * 1024 + num_nodes * 500)|string + "Ki" -%} +{% endif -%} + +apiVersion: v1 +kind: ReplicationController +metadata: + name: heapster-v1.0.0 + namespace: kube-system + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: heapster + template: + metadata: + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" + spec: + containers: + - image: gcr.io/google_containers/heapster:v1.0.0 + name: heapster + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: {{ metrics_memory }} + requests: + cpu: 100m + memory: {{ metrics_memory }} + command: + - /heapster + - --source=kubernetes.summary_api:'' + - --sink=influxdb:http://monitoring-influxdb:8086 + - --metric_resolution=60s + - image: gcr.io/google_containers/heapster:v1.0.0 + name: eventer + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: {{ eventer_memory }} + requests: + cpu: 100m + memory: {{ eventer_memory }} + command: + - /eventer + - --source=kubernetes:'' + - --sink=influxdb:http://monitoring-influxdb:8086 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/heapster-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/heapster-service.yaml new file mode 100644 index 0000000..e406d69 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/heapster-service.yaml @@ -0,0 +1,14 @@ +kind: Service +apiVersion: v1 +metadata: + name: heapster + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Heapster" +spec: + ports: + - port: 80 + targetPort: 8082 + selector: + k8s-app: heapster diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml new file mode 100644 index 0000000..4f75c8d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: monitoring-influxdb-grafana-v3 + namespace: kube-system + labels: + k8s-app: influxGrafana + version: v3 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: influxGrafana + version: v3 + template: + metadata: + labels: + k8s-app: influxGrafana + version: v3 + kubernetes.io/cluster-service: "true" + spec: + containers: + - image: gcr.io/google_containers/heapster_influxdb:v0.5 + name: influxdb + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 500Mi + requests: + cpu: 100m + memory: 500Mi + ports: + - containerPort: 8083 + hostPort: 8083 + - containerPort: 8086 + hostPort: 8086 + volumeMounts: + - name: influxdb-persistent-storage + mountPath: /data + - image: gcr.io/google_containers/heapster_grafana:v2.6.0-2 + name: grafana + env: + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + env: + # This variable is required to setup templates in Grafana. + - name: INFLUXDB_SERVICE_URL + value: http://monitoring-influxdb:8086 + # The following env variables are required to make Grafana accessible via + # the kubernetes api-server proxy. On production clusters, we recommend + # removing these env variables, setup auth for grafana, and expose the grafana + # service using a LoadBalancer or a public IP. + - name: GF_AUTH_BASIC_ENABLED + value: "false" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ORG_ROLE + value: Admin + - name: GF_SERVER_ROOT_URL + value: /api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/ + volumeMounts: + - name: grafana-persistent-storage + mountPath: /var + volumes: + - name: influxdb-persistent-storage + emptyDir: {} + - name: grafana-persistent-storage + emptyDir: {} + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/influxdb-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/influxdb-service.yaml new file mode 100644 index 0000000..066e052 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/influxdb/influxdb-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: monitoring-influxdb + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "InfluxDB" +spec: + ports: + - name: http + port: 8083 + targetPort: 8083 + - name: api + port: 8086 + targetPort: 8086 + selector: + k8s-app: influxGrafana + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/standalone/heapster-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/standalone/heapster-controller.yaml new file mode 100644 index 0000000..41f6196 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -0,0 +1,39 @@ +{% set metrics_memory = "200Mi" -%} +{% set num_nodes = pillar.get('num_nodes', -1) -%} +{% if num_nodes >= 0 -%} + {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%} +{% endif -%} + +apiVersion: v1 +kind: ReplicationController +metadata: + name: heapster-v1.0.0 + namespace: kube-system + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: heapster + template: + metadata: + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" + spec: + containers: + - image: gcr.io/google_containers/heapster:v1.0.0 + name: heapster + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: {{ metrics_memory }} + requests: + cpu: 100m + memory: {{ metrics_memory }} + command: + - /heapster + - --source=kubernetes.summary_api:'' + - --metric_resolution=60s diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/standalone/heapster-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/standalone/heapster-service.yaml new file mode 100644 index 0000000..31e8b96 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/cluster-monitoring/standalone/heapster-service.yaml @@ -0,0 +1,14 @@ +kind: Service +apiVersion: v1 +metadata: + name: heapster + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Heapster" +spec: + ports: + - port: 80 + targetPort: 8082 + selector: + k8s-app: heapster diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dashboard/dashboard-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dashboard/dashboard-controller.yaml new file mode 100644 index 0000000..5f69f3b --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dashboard/dashboard-controller.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + # Keep the name in sync with image version and + # gce/coreos/kube-manifests/addons/dashboard counterparts + name: kubernetes-dashboard-v1.0.0 + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: kubernetes-dashboard + image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.0.0 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + ports: + - containerPort: 9090 + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dashboard/dashboard-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dashboard/dashboard-service.yaml new file mode 100644 index 0000000..195b503 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dashboard/dashboard-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-dashboard + namespace: kube-system + labels: + k8s-app: kubernetes-dashboard + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: kubernetes-dashboard + ports: + - port: 80 + targetPort: 9090 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dns/skydns-rc.yaml.in b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dns/skydns-rc.yaml.in new file mode 100644 index 0000000..5185176 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dns/skydns-rc.yaml.in @@ -0,0 +1,129 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-dns-v11 + namespace: kube-system + labels: + k8s-app: kube-dns + version: v11 + kubernetes.io/cluster-service: "true" +spec: + replicas: {{ pillar['dns_replicas'] }} + selector: + k8s-app: kube-dns + version: v11 + template: + metadata: + labels: + k8s-app: kube-dns + version: v11 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: etcd + image: gcr.io/google_containers/etcd-amd64:2.2.1 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 500Mi + requests: + cpu: 100m + memory: 50Mi + command: + - /usr/local/bin/etcd + - -data-dir + - /var/etcd/data + - -listen-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -advertise-client-urls + - http://127.0.0.1:2379,http://127.0.0.1:4001 + - -initial-cluster-token + - skydns-etcd + volumeMounts: + - name: etcd-storage + mountPath: /var/etcd/data + - name: kube2sky + image: gcr.io/google_containers/kube2sky:1.14 + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + # Kube2sky watches all pods. + memory: 200Mi + requests: + cpu: 100m + memory: 50Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 30 + timeoutSeconds: 5 + args: + # command = "/kube2sky" + - --domain={{ pillar['dns_domain'] }} + - name: skydns + image: gcr.io/google_containers/skydns:2015-10-13-8c72f8c + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 100m + memory: 50Mi + args: + # command = "/skydns" + - -machines=http://127.0.0.1:4001 + - -addr=0.0.0.0:53 + - -ns-rotate=false + - -domain={{ pillar['dns_domain'] }}. + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: gcr.io/google_containers/exechealthz:1.0 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + args: + - -cmd=nslookup kubernetes.default.svc.{{ pillar['dns_domain'] }} 127.0.0.1 >/dev/null + - -port=8080 + ports: + - containerPort: 8080 + protocol: TCP + volumes: + - name: etcd-storage + emptyDir: {} + dnsPolicy: Default # Don't use cluster DNS. diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dns/skydns-svc.yaml.in b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dns/skydns-svc.yaml.in new file mode 100644 index 0000000..242c887 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/dns/skydns-svc.yaml.in @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: {{ pillar['dns_server'] }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/es-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/es-controller.yaml new file mode 100644 index 0000000..1c9e19c --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/es-controller.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: elasticsearch-logging-v1 + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + version: v1 + kubernetes.io/cluster-service: "true" +spec: + replicas: 2 + selector: + k8s-app: elasticsearch-logging + version: v1 + template: + metadata: + labels: + k8s-app: elasticsearch-logging + version: v1 + kubernetes.io/cluster-service: "true" + spec: + containers: + - image: gcr.io/google_containers/elasticsearch:1.8 + name: elasticsearch-logging + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + requests: + cpu: 100m + ports: + - containerPort: 9200 + name: db + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - name: es-persistent-storage + mountPath: /data + volumes: + - name: es-persistent-storage + emptyDir: {} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/es-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/es-service.yaml new file mode 100644 index 0000000..abf1fd3 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/es-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-logging + namespace: kube-system + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Elasticsearch" +spec: + ports: + - port: 9200 + protocol: TCP + targetPort: db + selector: + k8s-app: elasticsearch-logging diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/kibana-controller.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/kibana-controller.yaml new file mode 100644 index 0000000..f438bce --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/kibana-controller.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kibana-logging-v1 + namespace: kube-system + labels: + k8s-app: kibana-logging + version: v1 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kibana-logging + version: v1 + template: + metadata: + labels: + k8s-app: kibana-logging + version: v1 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: kibana-logging + image: gcr.io/google_containers/kibana:1.3 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + requests: + cpu: 100m + env: + - name: "ELASTICSEARCH_URL" + value: "http://elasticsearch-logging:9200" + ports: + - containerPort: 5601 + name: ui + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/kibana-service.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/kibana-service.yaml new file mode 100644 index 0000000..43efada --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/fluentd-elasticsearch/kibana-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: kibana-logging + namespace: kube-system + labels: + k8s-app: kibana-logging + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Kibana" +spec: + ports: + - port: 5601 + protocol: TCP + targetPort: ui + selector: + k8s-app: kibana-logging diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/init.sls new file mode 100644 index 0000000..3989edd --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/init.sls @@ -0,0 +1,218 @@ +addon-dir-delete: + file.absent: + - name: /etc/kubernetes/addons + +addon-dir-create: + file.directory: + - name: /etc/kubernetes/addons + - user: root + - group: root + - mode: 0755 + - require: + - file: addon-dir-delete + +/etc/kubernetes/addons/namespace.yaml: + file.managed: + - source: salt://kube-addons/namespace.yaml + - user: root + - group: root + - file_mode: 644 + +{% if pillar.get('enable_cluster_monitoring', '').lower() == 'influxdb' %} +/etc/kubernetes/addons/cluster-monitoring/influxdb: + file.recurse: + - source: salt://kube-addons/cluster-monitoring/influxdb + - include_pat: E@(^.+\.yaml$|^.+\.json$) + - template: jinja + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_l7_loadbalancing', '').lower() == 'glbc' %} +/etc/kubernetes/addons/cluster-loadbalancing/glbc: + file.recurse: + - source: salt://kube-addons/cluster-loadbalancing/glbc + - include_pat: E@(^.+\.yaml$|^.+\.json$) + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_cluster_monitoring', '').lower() == 'google' %} +/etc/kubernetes/addons/cluster-monitoring/google: + file.recurse: + - source: salt://kube-addons/cluster-monitoring/google + - include_pat: E@(^.+\.yaml$|^.+\.json$) + - template: jinja + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_cluster_monitoring', '').lower() == 'standalone' %} +/etc/kubernetes/addons/cluster-monitoring/standalone: + file.recurse: + - source: salt://kube-addons/cluster-monitoring/standalone + - include_pat: E@(^.+\.yaml$|^.+\.json$) + - template: jinja + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_cluster_monitoring', '').lower() == 'googleinfluxdb' %} +/etc/kubernetes/addons/cluster-monitoring/googleinfluxdb: + file.recurse: + - source: salt://kube-addons/cluster-monitoring + - include_pat: E@(^.+\.yaml$|^.+\.json$) + - exclude_pat: E@(^.+heapster-controller\.yaml$|^.+heapster-controller\.json$) + - template: jinja + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %} +/etc/kubernetes/addons/dns/skydns-svc.yaml: + file.managed: + - source: salt://kube-addons/dns/skydns-svc.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True + +/etc/kubernetes/addons/dns/skydns-rc.yaml: + file.managed: + - source: salt://kube-addons/dns/skydns-rc.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True +{% endif %} + +{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %} +/etc/kubernetes/addons/registry/registry-svc.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-svc.yaml + - user: root + - group: root + - file_mode: 644 + - makedirs: True + +/etc/kubernetes/addons/registry/registry-rc.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-rc.yaml + - user: root + - group: root + - file_mode: 644 + - makedirs: True + +/etc/kubernetes/addons/registry/registry-pv.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-pv.yaml.in + - template: jinja + - user: root + - group: root + - file_mode: 644 + - makedirs: True + +/etc/kubernetes/addons/registry/registry-pvc.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-pvc.yaml.in + - template: jinja + - user: root + - group: root + - file_mode: 644 + - makedirs: True +{% endif %} + +{% if pillar.get('enable_node_logging', '').lower() == 'true' + and pillar.get('logging_destination', '').lower() == 'elasticsearch' + and pillar.get('enable_cluster_logging', '').lower() == 'true' %} +/etc/kubernetes/addons/fluentd-elasticsearch: + file.recurse: + - source: salt://kube-addons/fluentd-elasticsearch + - include_pat: E@^.+\.yaml$ + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_cluster_ui', '').lower() == 'true' %} +/etc/kubernetes/addons/dashboard: + file.recurse: + - source: salt://kube-addons/dashboard + - include_pat: E@^.+\.yaml$ + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +/etc/kubernetes/kube-addons.sh: + file.managed: + - source: salt://kube-addons/kube-addons.sh + - user: root + - group: root + - mode: 755 + +/etc/kubernetes/kube-addon-update.sh: + file.managed: + - source: salt://kube-addons/kube-addon-update.sh + - user: root + - group: root + - mode: 755 + +{% if pillar.get('is_systemd') %} + +{{ pillar.get('systemd_system_path') }}/kube-addons.service: + file.managed: + - source: salt://kube-addons/kube-addons.service + - user: root + - group: root + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce kube-addons + - watch: + - file: {{ pillar.get('systemd_system_path') }}/kube-addons.service + +{% else %} + +/etc/init.d/kube-addons: + file.managed: + - source: salt://kube-addons/initd + - user: root + - group: root + - mode: 755 + +{% endif %} + +# Stop kube-addons service each time salt is executed, just in case +# there was a modification of addons. +# Actually, this should be handled by watching file changes, but +# somehow it doesn't work. +service-kube-addon-stop: + service.dead: + - name: kube-addons + +kube-addons: + service.running: + - enable: True + - require: + - service: service-kube-addon-stop + - watch: +{% if pillar.get('is_systemd') %} + - file: {{ pillar.get('systemd_system_path') }}/kube-addons.service +{% else %} + - file: /etc/init.d/kube-addons +{% endif %} +{% if pillar.get('is_systemd') %} + - provider: + - service: systemd +{%- endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/initd b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/initd new file mode 100644 index 0000000..c9abf85 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/initd @@ -0,0 +1,120 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kube-addons +# Required-Start: $local_fs $network $syslog kube-apiserver +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Kubernetes Addon Object Manager +# Description: +# Enforces installation of Kubernetes Addon Objects +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Kubernetes Addon Object Manager" +NAME=kube-addons +DAEMON_LOG_FILE=/var/log/${NAME}.log +PIDFILE=/var/run/${NAME}.pid +SCRIPTNAME=/etc/init.d/${NAME} +KUBE_ADDONS_SH=/etc/kubernetes/kube-addons.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + + + +# +# Function that starts the daemon/service +# +do_start() +{ + # use setsid to make sure the new daemon has its own group (I suppose + # start-stop-daemon does create a process group, but let's stay on the + # safe side). + setsid start-stop-daemon --start --verbose --background --no-close --make-pidfile --pidfile "${PIDFILE}" --startas "${KUBE_ADDONS_SH}" > ${DAEMON_LOG_FILE} 2>&1 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # start-stop-daemon is not used because we have to stop all children + # limitations: + # - stop does not work if the pid file is missing + # - stop does not work if the daemon process is missing (children will not + # be killed) + # This is sufficient - remaining processes will end after a while. + + local pid + pid=$(cat "${PIDFILE}" 2> /dev/null) + if [[ $? != 0 ]]; then + return 1 + fi + local pgrp + # find the process group for the service and kill entire group + # o - output format: pgpg - process group + pgrp=$(ps --no-headers --pid "${pid}" -o pgrp 2>/dev/null) + if [[ $? != 0 ]] || [[ "${pgrp}" == "" ]]; then + return 1 + fi + pgrp=$(echo -e ${pgrp}) # strip whitespaces (that's why there are no quotes around pgrp) + # negative pid is for killing entire group + kill -- -${pgrp} 2> /dev/null + if [[ $? != 0 ]]; then + return 2 + fi + rm -f "${PIDFILE}" + return +} + +case "$1" in + start) + log_daemon_msg "Starting ${DESC}" "${NAME}" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping ${DESC}" "${NAME}" + do_stop + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + status) + status_of_proc -p "${PIDFILE}" "${KUBE_ADDONS_SH}" "${NAME}" + ;; + + restart|force-reload) + log_daemon_msg "Restarting ${DESC}" "${NAME}" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: ${SCRIPTNAME} {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addon-update.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addon-update.sh new file mode 100755 index 0000000..6ba07b1 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addon-update.sh @@ -0,0 +1,513 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The business logic for whether a given object should be created +# was already enforced by salt, and /etc/kubernetes/addons is the +# managed result is of that. Start everything below that directory. + +# Parameters +# $1 path to add-ons + + +# LIMITATIONS +# 1. controllers are not updated unless their name is changed +# 3. Services will not be updated unless their name is changed, +# but for services we actually want updates without name change. +# 4. Json files are not handled at all. Currently addons must be +# in yaml files +# 5. exit code is probably not always correct (I haven't checked +# carefully if it works in 100% cases) +# 6. There are no unittests +# 8. Will not work if the total length of paths to addons is greater than +# bash can handle. Probably it is not a problem: ARG_MAX=2097152 on GCE. +# 9. Performance issue: yaml files are read many times in a single execution. + +# cosmetic improvements to be done +# 1. improve the log function; add timestamp, file name, etc. +# 2. logging doesn't work from files that print things out. +# 3. kubectl prints the output to stderr (the output should be captured and then +# logged) + + + +# global config +KUBECTL=${TEST_KUBECTL:-} # substitute for tests +KUBECTL=${KUBECTL:-${KUBECTL_BIN:-}} +KUBECTL=${KUBECTL:-/usr/local/bin/kubectl} +if [[ ! -x ${KUBECTL} ]]; then + echo "ERROR: kubectl command (${KUBECTL}) not found or is not executable" 1>&2 + exit 1 +fi + +# If an add-on definition is incorrect, or a definition has just disappeared +# from the local directory, the script will still keep on retrying. +# The script does not end until all retries are done, so +# one invalid manifest may block updates of other add-ons. +# Be careful how you set these parameters +NUM_TRIES=1 # will be updated based on input parameters +DELAY_AFTER_ERROR_SEC=${TEST_DELAY_AFTER_ERROR_SEC:=10} + + +# remember that you can't log from functions that print some output (because +# logs are also printed on stdout) +# $1 level +# $2 message +function log() { + # manage log levels manually here + + # add the timestamp if you find it useful + case $1 in + DB3 ) +# echo "$1: $2" + ;; + DB2 ) +# echo "$1: $2" + ;; + DBG ) +# echo "$1: $2" + ;; + INFO ) + echo "$1: $2" + ;; + WRN ) + echo "$1: $2" + ;; + ERR ) + echo "$1: $2" + ;; + * ) + echo "INVALID_LOG_LEVEL $1: $2" + ;; + esac +} + +#$1 yaml file path +function get-object-kind-from-file() { + # prints to stdout, so log cannot be used + #WARNING: only yaml is supported + cat $1 | ${PYTHON} -c ''' +try: + import pipes,sys,yaml + y = yaml.load(sys.stdin) + labels = y["metadata"]["labels"] + if ("kubernetes.io/cluster-service", "true") not in labels.iteritems(): + # all add-ons must have the label "kubernetes.io/cluster-service". + # Otherwise we are ignoring them (the update will not work anyway) + print "ERROR" + else: + print y["kind"] +except Exception, ex: + print "ERROR" + ''' +} + +# $1 yaml file path +# returns a string of the form / (we call it nsnames) +function get-object-nsname-from-file() { + # prints to stdout, so log cannot be used + #WARNING: only yaml is supported + #addons that do not specify a namespace are assumed to be in "default". + cat $1 | ${PYTHON} -c ''' +try: + import pipes,sys,yaml + y = yaml.load(sys.stdin) + labels = y["metadata"]["labels"] + if ("kubernetes.io/cluster-service", "true") not in labels.iteritems(): + # all add-ons must have the label "kubernetes.io/cluster-service". + # Otherwise we are ignoring them (the update will not work anyway) + print "ERROR" + else: + try: + print "%s/%s" % (y["metadata"]["namespace"], y["metadata"]["name"]) + except Exception, ex: + print "default/%s" % y["metadata"]["name"] +except Exception, ex: + print "ERROR" + ''' +} + +# $1 addon directory path +# $2 addon type (e.g. ReplicationController) +# echoes the string with paths to files containing addon for the given type +# works only for yaml files (!) (ignores json files) +function get-addon-paths-from-disk() { + # prints to stdout, so log cannot be used + local -r addon_dir=$1 + local -r obj_type=$2 + local kind + local file_path + for file_path in $(find ${addon_dir} -name \*.yaml); do + kind=$(get-object-kind-from-file ${file_path}) + # WARNING: assumption that the topmost indentation is zero (I'm not sure yaml allows for topmost indentation) + if [[ "${kind}" == "${obj_type}" ]]; then + echo ${file_path} + fi + done +} + +# waits for all subprocesses +# returns 0 if all of them were successful and 1 otherwise +function wait-for-jobs() { + local rv=0 + local pid + for pid in $(jobs -p); do + wait ${pid} + if [[ $? -ne 0 ]]; then + rv=1; + log ERR "error in pid ${pid}" + fi + log DB2 "pid ${pid} completed, current error code: ${rv}" + done + return ${rv} +} + + +function run-until-success() { + local -r command=$1 + local tries=$2 + local -r delay=$3 + local -r command_name=$1 + while [ ${tries} -gt 0 ]; do + log DBG "executing: '$command'" + # let's give the command as an argument to bash -c, so that we can use + # && and || inside the command itself + /bin/bash -c "${command}" && \ + log DB3 "== Successfully executed ${command_name} at $(date -Is) ==" && \ + return 0 + let tries=tries-1 + log INFO "== Failed to execute ${command_name} at $(date -Is). ${tries} tries remaining. ==" + sleep ${delay} + done + return 1 +} + +# $1 object type +# returns a list of / pairs (nsnames) +function get-addon-nsnames-from-server() { + local -r obj_type=$1 + "${KUBECTL}" get "${obj_type}" --all-namespaces -o go-template="{{range.items}}{{.metadata.namespace}}/{{.metadata.name}} {{end}}" --api-version=v1 -l kubernetes.io/cluster-service=true +} + +# returns the characters after the last separator (including) +# If the separator is empty or if it doesn't appear in the string, +# an empty string is printed +# $1 input string +# $2 separator (must be single character, or empty) +function get-suffix() { + # prints to stdout, so log cannot be used + local -r input_string=$1 + local -r separator=$2 + local suffix + + if [[ "${separator}" == "" ]]; then + echo "" + return + fi + + if [[ "${input_string}" == *"${separator}"* ]]; then + suffix=$(echo "${input_string}" | rev | cut -d "${separator}" -f1 | rev) + echo "${separator}${suffix}" + else + echo "" + fi +} + +# returns the characters up to the last '-' (without it) +# $1 input string +# $2 separator +function get-basename() { + # prints to stdout, so log cannot be used + local -r input_string=$1 + local -r separator=$2 + local suffix + suffix="$(get-suffix ${input_string} ${separator})" + # this will strip the suffix (if matches) + echo ${input_string%$suffix} +} + +function stop-object() { + local -r obj_type=$1 + local -r namespace=$2 + local -r obj_name=$3 + log INFO "Stopping ${obj_type} ${namespace}/${obj_name}" + + run-until-success "${KUBECTL} stop --namespace=${namespace} ${obj_type} ${obj_name}" ${NUM_TRIES} ${DELAY_AFTER_ERROR_SEC} +} + +function create-object() { + local -r obj_type=$1 + local -r file_path=$2 + + local nsname_from_file + nsname_from_file=$(get-object-nsname-from-file ${file_path}) + if [[ "${nsname_from_file}" == "ERROR" ]]; then + log INFO "Cannot read object name from ${file_path}. Ignoring" + return 1 + fi + IFS='/' read namespace obj_name <<< "${nsname_from_file}" + + log INFO "Creating new ${obj_type} from file ${file_path} in namespace ${namespace}, name: ${obj_name}" + # this will keep on failing if the ${file_path} disappeared in the meantime. + # Do not use too many retries. + run-until-success "${KUBECTL} create --namespace=${namespace} -f ${file_path}" ${NUM_TRIES} ${DELAY_AFTER_ERROR_SEC} +} + +function update-object() { + local -r obj_type=$1 + local -r namespace=$2 + local -r obj_name=$3 + local -r file_path=$4 + log INFO "updating the ${obj_type} ${namespace}/${obj_name} with the new definition ${file_path}" + stop-object ${obj_type} ${namespace} ${obj_name} + create-object ${obj_type} ${file_path} +} + +# deletes the objects from the server +# $1 object type +# $2 a list of object nsnames +function stop-objects() { + local -r obj_type=$1 + local -r obj_nsnames=$2 + local namespace + local obj_name + for nsname in ${obj_nsnames}; do + IFS='/' read namespace obj_name <<< "${nsname}" + stop-object ${obj_type} ${namespace} ${obj_name} & + done +} + +# creates objects from the given files +# $1 object type +# $2 a list of paths to definition files +function create-objects() { + local -r obj_type=$1 + local -r file_paths=$2 + local file_path + for file_path in ${file_paths}; do + # Remember that the file may have disappear by now + # But we don't want to check it here because + # such race condition may always happen after + # we check it. Let's have the race + # condition happen a bit more often so that + # we see that our tests pass anyway. + create-object ${obj_type} ${file_path} & + done +} + +# updates objects +# $1 object type +# $2 a list of update specifications +# each update specification is a ';' separated pair: ; +function update-objects() { + local -r obj_type=$1 # ignored + local -r update_spec=$2 + local objdesc + local nsname + local obj_name + local namespace + + for objdesc in ${update_spec}; do + IFS=';' read nsname file_path <<< "${objdesc}" + IFS='/' read namespace obj_name <<< "${nsname}" + + update-object ${obj_type} ${namespace} ${obj_name} ${file_path} & + done +} + +# Global variables set by function match-objects. +nsnames_for_delete="" # a list of object nsnames to be deleted +for_update="" # a list of pairs ; for objects that should be updated +nsnames_for_ignore="" # a list of object nsnames that will be ignored +new_files="" # a list of file paths that weren't matched by any existing objects (these objects must be created now) + + +# $1 path to files with objects +# $2 object type in the API (ReplicationController or Service) +# $3 name separator (single character or empty) +function match-objects() { + local -r addon_dir=$1 + local -r obj_type=$2 + local -r separator=$3 + + # output variables (globals) + nsnames_for_delete="" + for_update="" + nsnames_for_ignore="" + new_files="" + + addon_nsnames_on_server=$(get-addon-nsnames-from-server "${obj_type}") + # if the api server is unavailable then abandon the update for this cycle + if [[ $? -ne 0 ]]; then + log ERR "unable to query ${obj_type} - exiting" + exit 1 + fi + + addon_paths_in_files=$(get-addon-paths-from-disk "${addon_dir}" "${obj_type}") + + log DB2 "addon_nsnames_on_server=${addon_nsnames_on_server}" + log DB2 "addon_paths_in_files=${addon_paths_in_files}" + + local matched_files="" + + local basensname_on_server="" + local nsname_on_server="" + local suffix_on_server="" + local nsname_from_file="" + local suffix_from_file="" + local found=0 + local addon_path="" + + # objects that were moved between namespaces will have different nsname + # because the namespace is included. So they will be treated + # like different objects and not updated but deleted and created again + # (in the current version update is also delete+create, so it does not matter) + for nsname_on_server in ${addon_nsnames_on_server}; do + basensname_on_server=$(get-basename ${nsname_on_server} ${separator}) + suffix_on_server="$(get-suffix ${nsname_on_server} ${separator})" + + log DB3 "Found existing addon ${nsname_on_server}, basename=${basensname_on_server}" + + # check if the addon is present in the directory and decide + # what to do with it + # this is not optimal because we're reading the files over and over + # again. But for small number of addons it doesn't matter so much. + found=0 + for addon_path in ${addon_paths_in_files}; do + nsname_from_file=$(get-object-nsname-from-file ${addon_path}) + if [[ "${nsname_from_file}" == "ERROR" ]]; then + log INFO "Cannot read object name from ${addon_path}. Ignoring" + continue + else + log DB2 "Found object name '${nsname_from_file}' in file ${addon_path}" + fi + suffix_from_file="$(get-suffix ${nsname_from_file} ${separator})" + + log DB3 "matching: ${basensname_on_server}${suffix_from_file} == ${nsname_from_file}" + if [[ "${basensname_on_server}${suffix_from_file}" == "${nsname_from_file}" ]]; then + log DB3 "matched existing ${obj_type} ${nsname_on_server} to file ${addon_path}; suffix_on_server=${suffix_on_server}, suffix_from_file=${suffix_from_file}" + found=1 + matched_files="${matched_files} ${addon_path}" + if [[ "${suffix_on_server}" == "${suffix_from_file}" ]]; then + nsnames_for_ignore="${nsnames_for_ignore} ${nsname_from_file}" + else + for_update="${for_update} ${nsname_on_server};${addon_path}" + fi + break + fi + done + if [[ ${found} -eq 0 ]]; then + log DB2 "No definition file found for replication controller ${nsname_on_server}. Scheduling for deletion" + nsnames_for_delete="${nsnames_for_delete} ${nsname_on_server}" + fi + done + + log DB3 "matched_files=${matched_files}" + + + # note that if the addon file is invalid (or got removed after listing files + # but before we managed to match it) it will not be matched to any + # of the existing objects. So we will treat it as a new file + # and try to create its object. + for addon_path in ${addon_paths_in_files}; do + echo ${matched_files} | grep "${addon_path}" >/dev/null + if [[ $? -ne 0 ]]; then + new_files="${new_files} ${addon_path}" + fi + done +} + + + +function reconcile-objects() { + local -r addon_path=$1 + local -r obj_type=$2 + local -r separator=$3 # name separator + match-objects ${addon_path} ${obj_type} ${separator} + + log DBG "${obj_type}: nsnames_for_delete=${nsnames_for_delete}" + log DBG "${obj_type}: for_update=${for_update}" + log DBG "${obj_type}: nsnames_for_ignore=${nsnames_for_ignore}" + log DBG "${obj_type}: new_files=${new_files}" + + stop-objects "${obj_type}" "${nsnames_for_delete}" + # wait for jobs below is a protection against changing the basename + # of a replication controllerm without changing the selector. + # If we don't wait, the new rc may be created before the old one is deleted + # In such case the old one will wait for all its pods to be gone, but the pods + # are created by the new replication controller. + # passing --cascade=false could solve the problem, but we want + # all orphan pods to be deleted. + wait-for-jobs + stopResult=$? + + create-objects "${obj_type}" "${new_files}" + update-objects "${obj_type}" "${for_update}" + + local nsname + for nsname in ${nsnames_for_ignore}; do + log DB2 "The ${obj_type} ${nsname} is already up to date" + done + + wait-for-jobs + createUpdateResult=$? + + if [[ ${stopResult} -eq 0 ]] && [[ ${createUpdateResult} -eq 0 ]]; then + return 0 + else + return 1 + fi +} + +function update-addons() { + local -r addon_path=$1 + # be careful, reconcile-objects uses global variables + reconcile-objects ${addon_path} ReplicationController "-" & + + # We don't expect names to be versioned for the following kinds, so + # we match the entire name, ignoring version suffix. + # That's why we pass an empty string as the version separator. + # If the description differs on disk, the object should be recreated. + # This is not implemented in this version. + reconcile-objects ${addon_path} Service "" & + reconcile-objects ${addon_path} PersistentVolume "" & + reconcile-objects ${addon_path} PersistentVolumeClaim "" & + + wait-for-jobs + if [[ $? -eq 0 ]]; then + log INFO "== Kubernetes addon update completed successfully at $(date -Is) ==" + else + log WRN "== Kubernetes addon update completed with errors at $(date -Is) ==" + fi +} + +# input parameters: +# $1 input directory +# $2 retry period in seconds - the script will retry api-server errors for approximately +# this amound of time (it is not very precise), at interval equal $DELAY_AFTER_ERROR_SEC. +# + +if [[ $# -ne 2 ]]; then + echo "Illegal number of parameters. Usage $0 addon-dir [retry-period]" 1>&2 + exit 1 +fi + +NUM_TRIES=$(($2 / ${DELAY_AFTER_ERROR_SEC})) +if [[ ${NUM_TRIES} -le 0 ]]; then + NUM_TRIES=1 +fi + +addon_path=$1 +update-addons ${addon_path} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addons.service b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addons.service new file mode 100644 index 0000000..293ca10 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addons.service @@ -0,0 +1,9 @@ +[Unit] +Description=Kubernetes Addon Object Manager +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/etc/kubernetes/kube-addons.sh + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addons.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addons.sh new file mode 100644 index 0000000..19fafb6 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/kube-addons.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The business logic for whether a given object should be created +# was already enforced by salt, and /etc/kubernetes/addons is the +# managed result is of that. Start everything below that directory. +KUBECTL=${KUBECTL_BIN:-/usr/local/bin/kubectl} + +ADDON_CHECK_INTERVAL_SEC=${TEST_ADDON_CHECK_INTERVAL_SEC:-600} + +SYSTEM_NAMESPACE=kube-system +token_dir=${TOKEN_DIR:-/srv/kubernetes} +trusty_master=${TRUSTY_MASTER:-false} + +function ensure_python() { + if ! python --version > /dev/null 2>&1; then + echo "No python on the machine, will use a python image" + local -r PYTHON_IMAGE=python:2.7-slim-pyyaml + export PYTHON="docker run --interactive --rm --net=none ${PYTHON_IMAGE} python" + else + export PYTHON=python + fi +} + +function create-kubeconfig-secret() { + local -r token=$1 + local -r username=$2 + local -r server=$3 + local -r safe_username=$(tr -s ':_' '--' <<< "${username}") + + # Make a kubeconfig file with the token. + if [[ ! -z "${CA_CERT:-}" ]]; then + # If the CA cert is available, put it into the secret rather than using + # insecure-skip-tls-verify. + read -r -d '' kubeconfig </dev/null + rc=$? + if [[ "$rc" == 124 ]]; then + restart_docker=true + elif [[ "$rc" != 0 ]]; then + success=false + fi + done + if [[ "$success" == "true" ]]; then break; fi + if [[ "$restart_docker" == "true" ]]; then service docker restart; fi + sleep 15 + done +} + +# The business logic for whether a given object should be created +# was already enforced by salt, and /etc/kubernetes/addons is the +# managed result is of that. Start everything below that directory. +echo "== Kubernetes addon manager started at $(date -Is) with ADDON_CHECK_INTERVAL_SEC=${ADDON_CHECK_INTERVAL_SEC} ==" + +# Load any images that we may need. This is not needed for trusty master and +# the way it restarts docker daemon does not work for trusty. +if [[ "${trusty_master}" == "false" ]]; then + load-docker-images /srv/salt/kube-addons-images +fi + +ensure_python + +# Load the kube-env, which has all the environment variables we care +# about, in a flat yaml format. +kube_env_yaml="/var/cache/kubernetes-install/kube_env.yaml" +if [ ! -e "${kubelet_kubeconfig_file}" ]; then + eval $(${PYTHON} -c ''' +import pipes,sys,yaml + +for k,v in yaml.load(sys.stdin).iteritems(): + print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))) +''' < "${kube_env_yaml}") +fi + + +# Create the namespace that will be used to host the cluster-level add-ons. +start_addon /etc/kubernetes/addons/namespace.yaml 100 10 "" & + +# Wait for the default service account to be created in the kube-system namespace. +token_found="" +while [ -z "${token_found}" ]; do + sleep .5 + token_found=$(${KUBECTL} get --namespace="${SYSTEM_NAMESPACE}" serviceaccount default -o go-template="{{with index .secrets 0}}{{.name}}{{end}}" || true) +done + +echo "== default service account in the ${SYSTEM_NAMESPACE} namespace has token ${token_found} ==" + +# Generate secrets for "internal service accounts". +# TODO(etune): move to a completely yaml/object based +# workflow so that service accounts can be created +# at the same time as the services that use them. +# NOTE: needs to run as root to read this file. +# Read each line in the csv file of tokens. +# Expect errors when the script is started again. +# NOTE: secrets are created asynchronously, in background. +while read line; do + # Split each line into the token and username. + IFS=',' read -a parts <<< "${line}" + token=${parts[0]} + username=${parts[1]} + # DNS is special, since it's necessary for cluster bootstrapping. + if [[ "${username}" == "system:dns" ]] && [[ ! -z "${KUBERNETES_MASTER_NAME:-}" ]]; then + create-kubeconfig-secret "${token}" "${username}" "https://${KUBERNETES_MASTER_NAME}" + else + # Set the server to https://kubernetes. Pods/components that + # do not have DNS available will have to override the server. + create-kubeconfig-secret "${token}" "${username}" "https://kubernetes.default" + fi +done < "${token_dir}/known_tokens.csv" + +# Create admission_control objects if defined before any other addon services. If the limits +# are defined in a namespace other than default, we should still create the limits for the +# default namespace. +for obj in $(find /etc/kubernetes/admission-controls \( -name \*.yaml -o -name \*.json \)); do + start_addon "${obj}" 100 10 default & + echo "++ obj ${obj} is created ++" +done + +# Check if the configuration has changed recently - in case the user +# created/updated/deleted the files on the master. +while true; do + start_sec=$(date +"%s") + #kube-addon-update.sh must be deployed in the same directory as this file + `dirname $0`/kube-addon-update.sh /etc/kubernetes/addons ${ADDON_CHECK_INTERVAL_SEC} + end_sec=$(date +"%s") + len_sec=$((${end_sec}-${start_sec})) + # subtract the time passed from the sleep time + if [[ ${len_sec} -lt ${ADDON_CHECK_INTERVAL_SEC} ]]; then + sleep_time=$((${ADDON_CHECK_INTERVAL_SEC}-${len_sec})) + sleep ${sleep_time} + fi +done diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/namespace.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/namespace.yaml new file mode 100644 index 0000000..986f4b4 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/auth/registry-auth-rc.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/auth/registry-auth-rc.yaml new file mode 100644 index 0000000..1af623d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/auth/registry-auth-rc.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-registry-v0 + namespace: kube-system + labels: + k8s-app: kube-registry + version: v0 +# kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-registry + version: v0 + template: + metadata: + labels: + k8s-app: kube-registry + version: v0 +# kubernetes.io/cluster-service: "true" + spec: + containers: + - name: registry + image: registry:2 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + env: + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + - name: REGISTRY_AUTH_HTPASSWD_REALM + value: basic_realm + - name: REGISTRY_AUTH_HTPASSWD_PATH + value: /auth/htpasswd + volumeMounts: + - name: image-store + mountPath: /var/lib/registry + - name: auth-dir + mountPath: /auth + ports: + - containerPort: 5000 + name: registry + protocol: TCP + volumes: + - name: image-store + emptyDir: {} + - name: auth-dir + secret: + secretName: registry-auth-secret \ No newline at end of file diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/gcs/registry-gcs-rc.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/gcs/registry-gcs-rc.yaml new file mode 100644 index 0000000..e697403 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/gcs/registry-gcs-rc.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-registry-v0 + namespace: kube-system + labels: + k8s-app: kube-registry + version: v0 +# kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-registry + version: v0 + template: + metadata: + labels: + k8s-app: kube-registry + version: v0 +# kubernetes.io/cluster-service: "true" + spec: + containers: + - name: registry + image: registry:2 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + env: + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_STORAGE + value: gcs + - name: REGISTRY_STORAGE_GCS_BUCKET + value: kube-registry + - name: REGISTRY_STORAGE_GCS_KEYFILE + value: /gcs/keyfile + ports: + - containerPort: 5000 + name: registry + protocol: TCP + volumeMounts: + - name: gcs-key + mountPath: /gcs + volumes: + - name: gcs-key + secret: + secretName: gcs-key-secret diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-pv.yaml.in b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-pv.yaml.in new file mode 100644 index 0000000..b6fd2b3 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-pv.yaml.in @@ -0,0 +1,16 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: kube-system-kube-registry-pv + labels: + kubernetes.io/cluster-service: "true" +spec: +{% if pillar.get('cluster_registry_disk_type', '') == 'gce' %} + capacity: + storage: {{ pillar['cluster_registry_disk_size'] }} + accessModes: + - ReadWriteOnce + gcePersistentDisk: + pdName: "{{ pillar['cluster_registry_disk_name'] }}" + fsType: "ext4" +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-pvc.yaml.in b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-pvc.yaml.in new file mode 100644 index 0000000..92bfc69 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-pvc.yaml.in @@ -0,0 +1,13 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kube-registry-pvc + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ pillar['cluster_registry_disk_size'] }} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-rc.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-rc.yaml new file mode 100644 index 0000000..31df8ad --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-rc.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-registry-v0 + namespace: kube-system + labels: + k8s-app: kube-registry + version: v0 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-registry + version: v0 + template: + metadata: + labels: + k8s-app: kube-registry + version: v0 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: registry + image: registry:2 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + env: + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + volumeMounts: + - name: image-store + mountPath: /var/lib/registry + ports: + - containerPort: 5000 + name: registry + protocol: TCP + volumes: + - name: image-store + persistentVolumeClaim: + claimName: kube-registry-pvc diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-svc.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-svc.yaml new file mode 100644 index 0000000..b9f1cc4 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/registry-svc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-registry + namespace: kube-system + labels: + k8s-app: kube-registry + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeRegistry" +spec: + selector: + k8s-app: kube-registry + ports: + - name: registry + port: 5000 + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/tls/registry-tls-rc.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/tls/registry-tls-rc.yaml new file mode 100644 index 0000000..c2411c0 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/tls/registry-tls-rc.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-registry-v0 + namespace: kube-system + labels: + k8s-app: kube-registry + version: v0 +# kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-registry + version: v0 + template: + metadata: + labels: + k8s-app: kube-registry + version: v0 +# kubernetes.io/cluster-service: "true" + spec: + containers: + - name: registry + image: registry:2 + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + env: + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /certs/domain.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /certs/domain.key + volumeMounts: + - name: image-store + mountPath: /var/lib/registry + - name: cert-dir + mountPath: /certs + ports: + - containerPort: 5000 + name: registry + protocol: TCP + volumes: + - name: image-store + emptyDir: {} + - name: cert-dir + secret: + secretName: registry-tls-secret + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/tls/registry-tls-svc.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/tls/registry-tls-svc.yaml new file mode 100644 index 0000000..a9d59f1 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-addons/registry/tls/registry-tls-svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-registry + namespace: kube-system + labels: + k8s-app: kube-registry +# kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeRegistry" +spec: + selector: + k8s-app: kube-registry + type: LoadBalancer + ports: + - name: registry + port: 5000 + protocol: TCP diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-admission-controls/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-admission-controls/init.sls new file mode 100644 index 0000000..55cfd01 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-admission-controls/init.sls @@ -0,0 +1,10 @@ +{% if 'LimitRanger' in pillar.get('admission_control', '') %} +/etc/kubernetes/admission-controls/limit-range: + file.recurse: + - source: salt://kube-admission-controls/limit-range + - include_pat: E@(^.+\.yaml$|^.+\.json$) + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-admission-controls/limit-range/limit-range.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-admission-controls/limit-range/limit-range.yaml new file mode 100644 index 0000000..aff57ae --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-admission-controls/limit-range/limit-range.yaml @@ -0,0 +1,10 @@ +apiVersion: "v1" +kind: "LimitRange" +metadata: + name: "limits" + namespace: default +spec: + limits: + - type: "Container" + defaultRequest: + cpu: "100m" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-apiserver/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-apiserver/init.sls new file mode 100644 index 0000000..60c95a1 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-apiserver/init.sls @@ -0,0 +1,61 @@ +{% if grains.cloud is defined %} +{% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] %} +# TODO: generate and distribute tokens on other cloud providers. +/srv/kubernetes/known_tokens.csv: + file.managed: + - source: salt://kube-apiserver/known_tokens.csv + - user: root + - group: root + - mode: 600 +# - watch_in: +# - service: kube-apiserver +{% endif %} +{% endif %} + +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ,'vsphere'] %} +/srv/kubernetes/basic_auth.csv: + file.managed: + - source: salt://kube-apiserver/basic_auth.csv + - user: root + - group: root + - mode: 600 +{% endif %} + +/var/log/kube-apiserver.log: + file.managed: + - user: root + - group: root + - mode: 644 + +# Copy kube-apiserver manifest to manifests folder for kubelet. +# Current containervm image by default has both docker and kubelet +# running. But during cluster creation stage, docker and kubelet +# could be overwritten completely, or restarted due to flag changes. +# The ordering of salt states for service docker, kubelet and +# master-addon below is very important to avoid the race between +# salt restart docker or kubelet and kubelet start master components. +# Without the ordering of salt states, when gce instance boot up, +# configure-vm.sh will run and download the release. At the end of +# boot, run-salt will installs kube-apiserver.manifest files to +# kubelet config directory before the installation of proper version +# kubelet. Please see +# http://issue.k8s.io/10122#issuecomment-114566063 +# for detail explanation on this very issue. +/etc/kubernetes/manifests/kube-apiserver.manifest: + file.managed: + - source: salt://kube-apiserver/kube-apiserver.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - require: + - service: docker + - service: kubelet + +#stop legacy kube-apiserver service +stop_kube-apiserver: + service.dead: + - name: kube-apiserver + - enable: None diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-apiserver/kube-apiserver.manifest new file mode 100644 index 0000000..52b0089 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -0,0 +1,206 @@ +{% set daemon_args = "$DAEMON_ARGS" -%} +{% if grains['os_family'] == 'RedHat' -%} + {% set daemon_args = "" -%} +{% endif -%} + +{% set cloud_provider = "" -%} +{% set cloud_config = "" -%} +{% set cloud_config_mount = "" -%} +{% set cloud_config_volume = "" -%} +{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%} +{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%} + +{% set srv_kube_path = "/srv/kubernetes" -%} +{% set srv_sshproxy_path = "/srv/sshproxy" -%} + +{% if grains.cloud is defined -%} + {% if grains.cloud not in ['vagrant', 'vsphere'] -%} + {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} + {% endif -%} + + {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} + {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} + {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%} + {% endif -%} +{% endif -%} + +{% set advertise_address = "" -%} +{% if grains.advertise_address is defined -%} + {% set advertise_address = "--advertise-address=" + grains.advertise_address -%} +{% endif -%} + +{% set proxy_ssh_options = "" -%} +{% if grains.proxy_ssh_user is defined -%} + {% set proxy_ssh_options = "--ssh-user=" + grains.proxy_ssh_user + " --ssh-keyfile=/srv/sshproxy/.sshkeyfile" -%} + {# Append 40 characters onto command to work around #9822. #} + {# If mount list changes, this may also need to change. #} + {% set proxy_ssh_options = proxy_ssh_options + " " -%} +{% endif -%} + +{% set address = "--address=127.0.0.1" -%} + +{% set bind_address = "" -%} +{% if grains.publicAddressOverride is defined -%} + {% set bind_address = "--bind-address=" + grains.publicAddressOverride -%} +{% endif -%} + +{% set etcd_servers = "--etcd-servers=http://127.0.0.1:4001" -%} +{% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%} + +{% set service_cluster_ip_range = "" -%} +{% if pillar['service_cluster_ip_range'] is defined -%} + {% set service_cluster_ip_range = "--service-cluster-ip-range=" + pillar['service_cluster_ip_range'] -%} +{% endif -%} + +{% set cert_file = "--tls-cert-file=/srv/kubernetes/server.cert" -%} +{% set key_file = "--tls-private-key-file=/srv/kubernetes/server.key" -%} +{% set client_ca_file = "" -%} + +{% set secure_port = "6443" -%} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ] %} + {% set secure_port = "443" -%} + {% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%} +{% endif -%} + +{% set token_auth_file = "--token-auth-file=/dev/null" -%} +{% set basic_auth_file = "" -%} + +{% set min_request_timeout = "" -%} +{% if grains.minRequestTimeout is defined -%} + {% set min_request_timeout = "--min-request-timeout=" + grains.minRequestTimeout -%} +{% endif -%} + +{% if grains.cloud is defined -%} +{% if grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ] -%} + {% set token_auth_file = "--token-auth-file=/srv/kubernetes/known_tokens.csv" -%} +{% endif -%} +{% endif -%} + +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere'] %} + {% set basic_auth_file = "--basic-auth-file=/srv/kubernetes/basic_auth.csv" -%} +{% endif -%} + +{% set admission_control = "" -%} +{% if pillar['admission_control'] is defined -%} + {% set admission_control = "--admission-control=" + pillar['admission_control'] -%} +{% endif -%} + +{% set runtime_config = "" -%} +{% if grains.runtime_config is defined -%} + {% set runtime_config = "--runtime-config=" + grains.runtime_config -%} +{% endif -%} + +{% set log_level = pillar['log_level'] -%} +{% if pillar['api_server_test_log_level'] is defined -%} + {% set log_level = pillar['api_server_test_log_level'] -%} +{% endif -%} + +{% set params = address + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + " " + basic_auth_file + " " + min_request_timeout -%} +{% set params = params + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + " " + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options -%} + +# test_args has to be kept at the end, so they'll overwrite any prior configuration +{% if pillar['apiserver_test_args'] is defined -%} +{% set params = params + " " + pillar['apiserver_test_args'] -%} +{% endif -%} + +{ +"apiVersion": "v1", +"kind": "Pod", +"metadata": { + "name":"kube-apiserver", + "namespace": "kube-system" +}, +"spec":{ +"hostNetwork": true, +"containers":[ + { + "name": "kube-apiserver", + "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}", + "resources": { + "requests": { + "cpu": "250m" + } + }, + "command": [ + "/bin/sh", + "-c", + "/usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1" + ], + "livenessProbe": { + "httpGet": { + "host": "127.0.0.1", + "port": 8080, + "path": "/healthz" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 15 + }, + "ports":[ + { "name": "https", + "containerPort": {{secure_port}}, + "hostPort": {{secure_port}}},{ + "name": "local", + "containerPort": 8080, + "hostPort": 8080} + ], + "volumeMounts": [ + {{cloud_config_mount}} + {{additional_cloud_config_mount}} + { "name": "srvkube", + "mountPath": "{{srv_kube_path}}", + "readOnly": true}, + { "name": "logfile", + "mountPath": "/var/log/kube-apiserver.log", + "readOnly": false}, + { "name": "etcssl", + "mountPath": "/etc/ssl", + "readOnly": true}, + { "name": "varssl", + "mountPath": "/var/ssl", + "readOnly": true}, + { "name": "etcopenssl", + "mountPath": "/etc/openssl", + "readOnly": true}, + { "name": "etcpkitls", + "mountPath": "/etc/pki/tls", + "readOnly": true}, + { "name": "srvsshproxy", + "mountPath": "{{srv_sshproxy_path}}", + "readOnly": false} + ] + } +], +"volumes":[ + {{cloud_config_volume}} + {{additional_cloud_config_volume}} + { "name": "srvkube", + "hostPath": { + "path": "{{srv_kube_path}}"} + }, + { "name": "logfile", + "hostPath": { + "path": "/var/log/kube-apiserver.log"} + }, + { "name": "etcssl", + "hostPath": { + "path": "/etc/ssl"} + }, + { "name": "varssl", + "hostPath": { + "path": "/var/ssl"} + }, + { "name": "etcopenssl", + "hostPath": { + "path": "/etc/openssl"} + }, + { "name": "etcpkitls", + "hostPath": { + "path": "/etc/pki/tls"} + }, + { "name": "srvsshproxy", + "hostPath": { + "path": "{{srv_sshproxy_path}}"} + } +] +}} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-client-tools.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-client-tools.sls new file mode 100644 index 0000000..3c05f11 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-client-tools.sls @@ -0,0 +1,6 @@ +/usr/local/bin/kubectl: + file.managed: + - source: salt://kube-bins/kubectl + - user: root + - group: root + - mode: 755 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-controller-manager/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-controller-manager/init.sls new file mode 100644 index 0000000..906ead6 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-controller-manager/init.sls @@ -0,0 +1,30 @@ +# Copy kube-controller-manager manifest to manifests folder for kubelet. +# The ordering of salt states for service docker, kubelet and +# master-addon below is very important to avoid the race between +# salt restart docker or kubelet and kubelet start master components. +# Please see http://issue.k8s.io/10122#issuecomment-114566063 +# for detail explanation on this very issue. +/etc/kubernetes/manifests/kube-controller-manager.manifest: + file.managed: + - source: salt://kube-controller-manager/kube-controller-manager.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - require: + - service: docker + - service: kubelet + +/var/log/kube-controller-manager.log: + file.managed: + - user: root + - group: root + - mode: 644 + +stop-legacy-kube_controller_manager: + service.dead: + - name: kube-controller-manager + - enable: None + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest new file mode 100644 index 0000000..3e857ac --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -0,0 +1,150 @@ +{% set cluster_name = "" -%} +{% set cluster_cidr = "" -%} +{% set allocate_node_cidrs = "" -%} +{% set terminated_pod_gc = "" -%} + + +{% if pillar['instance_prefix'] is defined -%} + {% set cluster_name = "--cluster-name=" + pillar['instance_prefix'] -%} +{% endif -%} +{% if pillar['cluster_cidr'] is defined and pillar['cluster_cidr'] != "" -%} + {% set cluster_cidr = "--cluster-cidr=" + pillar['cluster_cidr'] -%} +{% endif -%} +# When we're using flannel it is responsible for cidr allocation. +# This is expected to be a short-term compromise. +{% if pillar.get('network_provider', '').lower() == 'flannel' %} + {% set allocate_node_cidrs = "--allocate-node-cidrs=false" -%} +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + {% set allocate_node_cidrs = "--allocate-node-cidrs=true" -%} +{% elif pillar['allocate_node_cidrs'] is defined -%} + {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%} +{% endif -%} +{% if pillar['terminated_pod_gc_threshold'] is defined -%} + {% set terminated_pod_gc = "--terminated-pod-gc-threshold=" + pillar['terminated_pod_gc_threshold'] -%} +{% endif -%} + +{% set cloud_provider = "" -%} +{% set cloud_config = "" -%} +{% set cloud_config_mount = "" -%} +{% set cloud_config_volume = "" -%} +{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%} +{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%} +{% set srv_kube_path = "/srv/kubernetes" -%} + +{% if grains.cloud is defined -%} + {% if grains.cloud not in ['vagrant', 'vsphere'] -%} + {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} + {% endif -%} + {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%} + + {% if grains.cloud in [ 'aws', 'gce' ] and grains.cloud_config is defined -%} + {% set cloud_config = "--cloud-config=" + grains.cloud_config -%} + {% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%} + {% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\"}}," -%} + {% endif -%} +{% endif -%} + +{% set root_ca_file = "" -%} + +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ] %} + {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%} +{% endif -%} + +{% set log_level = pillar['log_level'] -%} +{% if pillar['controller_manager_test_log_level'] is defined -%} + {% set log_level = pillar['controller_manager_test_log_level'] -%} +{% endif -%} + +{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + terminated_pod_gc + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%} + + +# test_args has to be kept at the end, so they'll overwrite any prior configuration +{% if pillar['controller_manager_test_args'] is defined -%} +{% set params = params + " " + pillar['controller_manager_test_args'] -%} +{% endif -%} + +{ +"apiVersion": "v1", +"kind": "Pod", +"metadata": { + "name":"kube-controller-manager", + "namespace": "kube-system" +}, +"spec":{ +"hostNetwork": true, +"containers":[ + { + "name": "kube-controller-manager", + "image": "{{pillar['kube_docker_registry']}}/kube-controller-manager:{{pillar['kube-controller-manager_docker_tag']}}", + "resources": { + "requests": { + "cpu": "200m" + } + }, + "command": [ + "/bin/sh", + "-c", + "/usr/local/bin/kube-controller-manager {{params}} 1>>/var/log/kube-controller-manager.log 2>&1" + ], + "livenessProbe": { + "httpGet": { + "host": "127.0.0.1", + "port": 10252, + "path": "/healthz" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 15 + }, + "volumeMounts": [ + {{cloud_config_mount}} + {{additional_cloud_config_mount}} + { "name": "srvkube", + "mountPath": "{{srv_kube_path}}", + "readOnly": true}, + { "name": "logfile", + "mountPath": "/var/log/kube-controller-manager.log", + "readOnly": false}, + { "name": "etcssl", + "mountPath": "/etc/ssl", + "readOnly": true}, + { "name": "varssl", + "mountPath": "/var/ssl", + "readOnly": true}, + { "name": "etcopenssl", + "mountPath": "/etc/openssl", + "readOnly": true}, + { "name": "etcpkitls", + "mountPath": "/etc/pki/tls", + "readOnly": true} + ] + } +], +"volumes":[ + {{cloud_config_volume}} + {{additional_cloud_config_volume}} + { "name": "srvkube", + "hostPath": { + "path": "{{srv_kube_path}}"} + }, + { "name": "logfile", + "hostPath": { + "path": "/var/log/kube-controller-manager.log"} + }, + { "name": "etcssl", + "hostPath": { + "path": "/etc/ssl"} + }, + { "name": "varssl", + "hostPath": { + "path": "/var/ssl"} + }, + { "name": "etcopenssl", + "hostPath": { + "path": "/etc/openssl"} + }, + { "name": "etcpkitls", + "hostPath": { + "path": "/etc/pki/tls"} + } +] +}} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/init.sls new file mode 100644 index 0000000..9ae0a8d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/init.sls @@ -0,0 +1,52 @@ +/etc/kubernetes/kube-master-addons.sh: + file.managed: + - source: salt://kube-master-addons/kube-master-addons.sh + - user: root + - group: root + - mode: 755 + +# Used to restart kube-master-addons service each time salt is run +# Actually, it does not work (the service is not restarted), +# but master-addon service always terminates after it does it job, +# so it is (usually) not running and it will be started when +# salt is run. +# This salt state is not removed because there is a risk +# of introducing regression in 1.0. Please remove it afterwards. +# See also the salt config for kube-addons to see how to restart +# a service on demand. +master-docker-image-tags: + file.touch: + - name: /srv/pillar/docker-images.sls + +{% if pillar.get('is_systemd') %} + +{{ pillar.get('systemd_system_path') }}/kube-master-addons.service: + file.managed: + - source: salt://kube-master-addons/kube-master-addons.service + - user: root + - group: root + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce kube-master-addons + - watch: + - file: master-docker-image-tags + - file: /etc/kubernetes/kube-master-addons.sh + - file: {{ pillar.get('systemd_system_path') }}/kube-master-addons.service + +{% else %} + +/etc/init.d/kube-master-addons: + file.managed: + - source: salt://kube-master-addons/initd + - user: root + - group: root + - mode: 755 + +kube-master-addons: + service.running: + - enable: True + - restart: True + - watch: + - file: master-docker-image-tags + - file: /etc/kubernetes/kube-master-addons.sh + +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/initd b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/initd new file mode 100644 index 0000000..6fef265 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/initd @@ -0,0 +1,95 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kube-master-addons +# Required-Start: $local_fs $network $syslog docker +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Kubernetes Master Addon Object Manager +# Description: +# Enforces installation of Kubernetes Master Addon Objects +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Kubernetes Master Addon Object Manager" +NAME=kube-master-addons +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +KUBE_MASTER_ADDONS_SH=/etc/kubernetes/kube-master-addons.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + + + +# +# Function that starts the daemon/service +# +do_start() +{ + ${KUBE_MASTER_ADDONS_SH} >${DAEMON_LOG_FILE} 2>&1 & + echo $! > ${PIDFILE} + disown +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + kill $(cat ${PIDFILE}) + rm ${PIDFILE} + return +} + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + status_of_proc -p $PIDFILE $KUBE_MASTER_ADDONS_SH $NAME + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/kube-master-addons.service b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/kube-master-addons.service new file mode 100644 index 0000000..64858a7 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/kube-master-addons.service @@ -0,0 +1,9 @@ +[Unit] +Description=Kubernetes-Master Addon Object Manager +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/etc/kubernetes/kube-master-addons.sh + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/kube-master-addons.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/kube-master-addons.sh new file mode 100755 index 0000000..4041e8f --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-master-addons/kube-master-addons.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# loadedImageFlags is a bit-flag to track which docker images loaded successfully. +let loadedImageFlags=0 + +while true; do + restart_docker=false + + if which docker 1>/dev/null 2>&1; then + + timeout 30 docker load -i /srv/salt/kube-bins/kube-apiserver.tar 1>/dev/null 2>&1 + rc=$? + if [[ $rc == 0 ]]; then + let loadedImageFlags="$loadedImageFlags|1" + elif [[ $rc == 124 ]]; then + restart_docker=true + fi + + timeout 30 docker load -i /srv/salt/kube-bins/kube-scheduler.tar 1>/dev/null 2>&1 + rc=$? + if [[ $rc == 0 ]]; then + let loadedImageFlags="$loadedImageFlags|2" + elif [[ $rc == 124 ]]; then + restart_docker=true + fi + + timeout 30 docker load -i /srv/salt/kube-bins/kube-controller-manager.tar 1>/dev/null 2>&1 + rc=$? + if [[ $rc == 0 ]]; then + let loadedImageFlags="$loadedImageFlags|4" + elif [[ $rc == 124 ]]; then + restart_docker=true + fi + fi + + # required docker images got installed. exit while loop. + if [[ $loadedImageFlags == 7 ]]; then break; fi + + # Sometimes docker load hang, restart docker daemon resolve the issue + if [[ $restart_docker ]]; then + if ! service docker restart; then # Try systemctl if there's no service command. + systemctl restart docker + fi + fi + + # sleep for 15 seconds before attempting to load docker images again + sleep 15 + +done + +# Now exit. After kube-push, salt will notice that the service is down and it +# will start it and new docker images will be loaded. diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/init.sls new file mode 100644 index 0000000..03495d7 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/init.sls @@ -0,0 +1,71 @@ +/etc/kubernetes/kube-node-unpacker.sh: + file.managed: + - source: salt://kube-node-unpacker/kube-node-unpacker.sh + - makedirs: True + - user: root + - group: root + - mode: 755 + +{% if grains.cloud is defined and grains.cloud == 'gce' %} +node-docker-image-tags: + file.touch: + - name: /srv/pillar/docker-images.sls +{% else %} +kube-proxy-tar: + file.managed: + - name: /srv/salt/kube-bins/kube-proxy.tar + - source: salt://kube-bins/kube-proxy.tar + - makedirs: True + - user: root + - group: root + - mode: 644 +{% endif %} + +{% set is_helium = '0' %} +# Super annoying, the salt version on GCE is old enough that 'salt.cmd.run' +# isn't supported +{% if grains.cloud is defined and grains.cloud == 'aws' %} + # Salt has terrible problems with systemd on AWS too + {% set is_helium = '0' %} +{% endif %} +# Salt Helium doesn't support systemd modules for service running +{% if pillar.get('is_systemd') and is_helium == '0' %} + +{{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service: + file.managed: + - source: salt://kube-node-unpacker/kube-node-unpacker.service + - user: root + - group: root + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce kube-node-unpacker + - watch: +{% if grains.cloud is defined and grains.cloud == 'gce' %} + - file: node-docker-image-tags +{% else %} + - file: kube-proxy-tar +{% endif %} + - file: /etc/kubernetes/kube-node-unpacker.sh + - file: {{ pillar.get('systemd_system_path') }}/kube-node-unpacker.service + +{% else %} + +/etc/init.d/kube-node-unpacker: + file.managed: + - source: salt://kube-node-unpacker/initd + - user: root + - group: root + - mode: 755 + +kube-node-unpacker: + service.running: + - enable: True + - restart: True + - watch: +{% if grains.cloud is defined and grains.cloud == 'gce' %} + - file: node-docker-image-tags +{% else %} + - file: kube-proxy-tar +{% endif %} + - file: /etc/kubernetes/kube-node-unpacker.sh + +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/initd b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/initd new file mode 100755 index 0000000..13a6112 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/initd @@ -0,0 +1,95 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kube-node-unpacker +# Required-Start: $local_fs $network $syslog docker +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Kubernetes Node Unpacker +# Description: +# Unpacks docker images on Kubernetes nodes +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Kubernetes Node Unpacker" +NAME=kube-node-unpacker +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +KUBE_MASTER_ADDONS_SH=/etc/kubernetes/kube-node-unpacker.sh + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + + + + +# +# Function that starts the daemon/service +# +do_start() +{ + ${KUBE_MASTER_ADDONS_SH} >${DAEMON_LOG_FILE} 2>&1 & + echo $! > ${PIDFILE} + disown +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + kill $(cat ${PIDFILE}) + rm ${PIDFILE} + return +} + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + status_of_proc -p $PIDFILE $KUBE_MASTER_ADDONS_SH $NAME + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service new file mode 100644 index 0000000..4bbac62 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/kube-node-unpacker.service @@ -0,0 +1,9 @@ +[Unit] +Description=Kubernetes Node Unpacker +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/etc/kubernetes/kube-node-unpacker.sh + +[Install] +WantedBy=multi-user.target diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh new file mode 100755 index 0000000..38e2aee --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-node-unpacker/kube-node-unpacker.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# loadedImageFlags is a bit-flag to track which docker images loaded successfully. +let loadedImageFlags=0 + +while true; do + restart_docker=false + + if which docker 1>/dev/null 2>&1; then + + timeout 30 docker load -i /srv/salt/kube-bins/kube-proxy.tar 1>/dev/null 2>&1 + rc=$? + if [[ "${rc}" == 0 ]]; then + let loadedImageFlags="${loadedImageFlags}|1" + elif [[ "${rc}" == 124 ]]; then + restart_docker=true + fi + fi + + # required docker images got installed. exit while loop. + if [[ "${loadedImageFlags}" == 1 ]]; then break; fi + + # Sometimes docker load hang, restart docker daemon resolve the issue + if [[ "${restart_docker}" ]]; then service docker restart; fi + + # sleep for 15 seconds before attempting to load docker images again + sleep 15 + +done + +# Now exit. After kube-push, salt will notice that the service is down and it +# will start it and new docker images will be loaded. diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/init.sls new file mode 100644 index 0000000..3ca22cc --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/init.sls @@ -0,0 +1,35 @@ +/var/lib/kube-proxy/kubeconfig: + file.managed: + - source: salt://kube-proxy/kubeconfig + - user: root + - group: root + - mode: 400 + - makedirs: true + +# kube-proxy in a static pod +/etc/kubernetes/manifests/kube-proxy.manifest: + file.managed: + - source: salt://kube-proxy/kube-proxy.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - context: + cpurequest: '200m' + - require: + - service: docker + - service: kubelet + +/var/log/kube-proxy.log: + file.managed: + - user: root + - group: root + - mode: 644 + +#stop legacy kube-proxy service +stop_kube-proxy: + service.dead: + - name: kube-proxy + - enable: None diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/kube-proxy.manifest b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/kube-proxy.manifest new file mode 100644 index 0000000..ff46247 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -0,0 +1,62 @@ +{% set kubeconfig = "--kubeconfig=/var/lib/kube-proxy/kubeconfig" -%} +{% if grains.api_servers is defined -%} + {% set api_servers = "--master=https://" + grains.api_servers -%} +{% else -%} + {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} + {% set api_servers = "--master=https://" + ips[0][0] -%} +{% endif -%} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ] %} + {% set api_servers_with_port = api_servers -%} +{% else -%} + {% set api_servers_with_port = api_servers + ":6443" -%} +{% endif -%} +{% set test_args = "" -%} +{% if pillar['kubeproxy_test_args'] is defined -%} + {% set test_args=pillar['kubeproxy_test_args'] %} +{% endif -%} + +{% set log_level = pillar['log_level'] -%} +{% if pillar['kubeproxy_test_log_level'] is defined -%} + {% set log_level = pillar['kubeproxy_test_log_level'] -%} +{% endif -%} + +# kube-proxy podspec +apiVersion: v1 +kind: Pod +metadata: + name: kube-proxy + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-proxy + image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}} + resources: + requests: + cpu: {{ cpurequest }} + command: + - /bin/sh + - -c + - kube-proxy {{api_servers_with_port}} {{kubeconfig}} --resource-container="" {{log_level}} {{test_args}} 1>>/var/log/kube-proxy.log 2>&1 + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - mountPath: /var/log + name: varlog + readOnly: false + - mountPath: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + readOnly: false + volumes: + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host + - hostPath: + path: /var/lib/kube-proxy/kubeconfig + name: kubeconfig + - hostPath: + path: /var/log + name: varlog diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/kubeconfig b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-proxy/kubeconfig new file mode 100644 index 0000000..e69de29 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-registry-proxy/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-registry-proxy/init.sls new file mode 100644 index 0000000..cc5d416 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-registry-proxy/init.sls @@ -0,0 +1,8 @@ +/etc/kubernetes/manifests/kube-registry-proxy.yaml: + file.managed: + - source: salt://kube-registry-proxy/kube-registry-proxy.yaml + - user: root + - group: root + - mode: 644 + - makedirs: True + - dir_mode: 755 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml new file mode 100644 index 0000000..14f9225 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-registry-proxy + namespace: kube-system +spec: + containers: + - name: kube-registry-proxy + image: gcr.io/google_containers/kube-registry-proxy:0.3 + resources: + limits: + cpu: 100m + memory: 50Mi + env: + - name: REGISTRY_HOST + value: kube-registry.kube-system.svc.cluster.local + - name: REGISTRY_PORT + value: "5000" + - name: FORWARD_PORT + value: "5000" + ports: + - name: registry + containerPort: 5000 + hostPort: 5000 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-scheduler/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-scheduler/init.sls new file mode 100644 index 0000000..785d131 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-scheduler/init.sls @@ -0,0 +1,30 @@ +# Copy kube-scheduler manifest to manifests folder for kubelet. +# The ordering of salt states for service docker, kubelet and +# master-addon below is very important to avoid the race between +# salt restart docker or kubelet and kubelet start master components. +# Please see http://issue.k8s.io/10122#issuecomment-114566063 +# for detail explanation on this very issue. +/etc/kubernetes/manifests/kube-scheduler.manifest: + file.managed: + - source: salt://kube-scheduler/kube-scheduler.manifest + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true + - dir_mode: 755 + - require: + - service: docker + - service: kubelet + +/var/log/kube-scheduler.log: + file.managed: + - user: root + - group: root + - mode: 644 + +#stop legacy kube-scheduler service +stop_kube-scheduler: + service.dead: + - name: kube-scheduler + - enable: None diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-scheduler/kube-scheduler.manifest b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-scheduler/kube-scheduler.manifest new file mode 100644 index 0000000..7ae312a --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kube-scheduler/kube-scheduler.manifest @@ -0,0 +1,62 @@ +{% set params = "" -%} + +{% set log_level = pillar['log_level'] -%} +{% if pillar['scheduler_test_log_level'] is defined -%} + {% set log_level = pillar['scheduler_test_log_level'] -%} +{% endif -%} + +{% set params = params + log_level -%} + +# test_args has to be kept at the end, so they'll overwrite any prior configuration +{% if pillar['scheduler_test_args'] is defined -%} +{% set params = params + " " + pillar['scheduler_test_args'] -%} +{% endif -%} + +{ +"apiVersion": "v1", +"kind": "Pod", +"metadata": { + "name":"kube-scheduler", + "namespace": "kube-system" +}, +"spec":{ +"hostNetwork": true, +"containers":[ + { + "name": "kube-scheduler", + "image": "{{pillar['kube_docker_registry']}}/kube-scheduler:{{pillar['kube-scheduler_docker_tag']}}", + "resources": { + "requests": { + "cpu": "100m" + } + }, + "command": [ + "/bin/sh", + "-c", + "/usr/local/bin/kube-scheduler --master=127.0.0.1:8080 {{params}} 1>>/var/log/kube-scheduler.log 2>&1" + ], + "livenessProbe": { + "httpGet": { + "host": "127.0.0.1", + "port": 10251, + "path": "/healthz" + }, + "initialDelaySeconds": 15, + "timeoutSeconds": 15 + }, + "volumeMounts": [ + { + "name": "logfile", + "mountPath": "/var/log/kube-scheduler.log", + "readOnly": false + } + ] + } +], +"volumes":[ + { "name": "logfile", + "hostPath": { + "path": "/var/log/kube-scheduler.log"} + } +] +}} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/default b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/default new file mode 100644 index 0000000..0aa2ac5 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/default @@ -0,0 +1,193 @@ +{% set daemon_args = "$DAEMON_ARGS" -%} +{% if grains['os_family'] == 'RedHat' -%} + {% set daemon_args = "" -%} +{% endif -%} + +{% if grains.api_servers is defined -%} + {% set api_servers = "--api-servers=https://" + grains.api_servers -%} +{% elif grains.apiservers is defined -%} # TODO(remove after 0.16.0): Deprecated form + {% set api_servers = "--api-servers=https://" + grains.apiservers -%} +{% elif grains['roles'][0] == 'kubernetes-master' -%} + {% set master_ipv4 = salt['grains.get']('fqdn_ip4')[0] -%} + {% set api_servers = "--api-servers=https://" + master_ipv4 -%} +{% else -%} + {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%} + {% set api_servers = "--api-servers=https://" + ips[0][0] -%} +{% endif -%} + +# TODO: remove nginx for other cloud providers. +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ] %} + {% set api_servers_with_port = api_servers -%} +{% else -%} + {% set api_servers_with_port = api_servers + ":6443" -%} +{% endif -%} + +{% set master_kubelet_args = "" %} + +{% set debugging_handlers = "--enable-debugging-handlers=true" -%} + +{% set reconcile_cidr_args = "" -%} +{% if grains['roles'][0] == 'kubernetes-master' -%} + {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] -%} + + # Unless given a specific directive, disable registration for the kubelet + # running on the master. + {% if grains.kubelet_api_servers is defined -%} + {% set api_servers_with_port = "--api-servers=https://" + grains.kubelet_api_servers -%} + {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false" -%} + {% set reconcile_cidr_args = "--reconcile-cidr=false" -%} + {% else -%} + {% set api_servers_with_port = "" -%} + {% endif -%} + + # Disable the debugging handlers (/run and /exec) to prevent arbitrary + # code execution on the master. + # TODO(roberthbailey): Relax this constraint once the master is self-hosted. + {% set debugging_handlers = "--enable-debugging-handlers=false" -%} + {% endif -%} +{% endif -%} + +{% set cloud_provider = "" -%} +{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'vsphere'] -%} + {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} +{% endif -%} + +{% set config = "--config=/etc/kubernetes/manifests" -%} + +{% set manifest_url = "" -%} +{% set manifest_url_header = "" -%} +{% if pillar.get('enable_manifest_url', '').lower() == 'true' %} + {% set manifest_url = "--manifest-url=" + pillar['manifest_url'] + " --manifest-url-header=" + pillar['manifest_url_header'] -%} +{% endif -%} + +{% set hostname_override = "" -%} +{% if grains.hostname_override is defined -%} + {% set hostname_override = " --hostname-override=" + grains.hostname_override -%} +{% endif -%} + +{% set cluster_dns = "" %} +{% set cluster_domain = "" %} +{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %} + {% set cluster_dns = "--cluster-dns=" + pillar['dns_server'] %} + {% set cluster_domain = "--cluster-domain=" + pillar['dns_domain'] %} +{% endif %} + +{% set docker_root = "" -%} +{% if grains.docker_root is defined -%} + {% set docker_root = " --docker-root=" + grains.docker_root -%} +{% endif -%} + +{% set kubelet_root = "" -%} +{% if grains.kubelet_root is defined -%} + {% set kubelet_root = " --root-dir=" + grains.kubelet_root -%} +{% endif -%} + +{% set configure_cbr0 = "" -%} +{% if pillar['allocate_node_cidrs'] is defined -%} + {% set configure_cbr0 = "--configure-cbr0=" + pillar['allocate_node_cidrs'] -%} +{% endif -%} + +{% set non_masquerade_cidr = "" -%} +{% if pillar.get('non_masquerade_cidr','') -%} + {% set non_masquerade_cidr = "--non-masquerade-cidr=" + pillar.non_masquerade_cidr -%} +{% endif -%} + +# The master kubelet cannot wait for the flannel daemon because it is responsible +# for starting up the flannel server in a static pod. So even though the flannel +# daemon runs on the master, it doesn't hold up cluster bootstrap. All the pods +# on the master run with host networking, so the master flannel doesn't care +# even if the network changes. We only need it for the master proxy. +{% set experimental_flannel_overlay = "" -%} +{% if pillar.get('network_provider', '').lower() == 'flannel' and grains['roles'][0] != 'kubernetes-master' %} + {% set experimental_flannel_overlay = "--experimental-flannel-overlay=true" %} +{% endif -%} + +# Setup cgroups hierarchies. +{% set cgroup_root = "" -%} +{% set system_container = "" -%} +{% set kubelet_container = "" -%} +{% set runtime_container = "" -%} +{% if grains['os_family'] == 'Debian' -%} + {% if pillar.get('is_systemd') %} + {% set cgroup_root = "--cgroup-root=docker" -%} + {% else %} + {% set cgroup_root = "--cgroup-root=/" -%} + {% set system_container = "--system-cgroups=/system" -%} + {% set runtime_container = "--runtime-cgroups=/docker-daemon" -%} + {% set kubelet_container= "--kubelet-cgroups=/kubelet" -%} + {% endif %} +{% endif -%} +{% if grains['oscodename'] in ['vivid','wily'] -%} + {% set cgroup_root = "--cgroup-root=docker" -%} +{% endif -%} + +{% set pod_cidr = "" %} +{% if grains['roles'][0] == 'kubernetes-master' %} + {% if grains.get('cbr-cidr') %} + {% set pod_cidr = "--pod-cidr=" + grains['cbr-cidr'] %} + {% elif api_servers_with_port == '' and pillar.get('network_provider', '').lower() == 'kubenet' %} + # Kubelet standalone mode needs a PodCIDR since there is no controller-manager + {% set pod_cidr = "--pod-cidr=10.76.0.0/16" %} + {% endif -%} +{% endif %} + +{% set cpu_cfs_quota = "" %} +{% if pillar['enable_cpu_cfs_quota'] is defined -%} + {% set cpu_cfs_quota = "--cpu-cfs-quota=" + pillar['enable_cpu_cfs_quota'] -%} +{% endif -%} + +{% set test_args = "" -%} +{% if pillar['kubelet_test_args'] is defined -%} + {% set test_args=pillar['kubelet_test_args'] %} +{% endif -%} + +{% set network_plugin = "" -%} +{% if pillar.get('network_provider', '').lower() == 'opencontrail' %} + {% set network_plugin = "--network-plugin=opencontrail" %} +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + {% set network_plugin = "--network-plugin=kubenet" -%} + {% if reconcile_cidr_args == '' -%} + {% set reconcile_cidr_args = "--reconcile-cidr=true" -%} + {% endif -%} +{% endif -%} + +# Don't pipe the --hairpin-mode flag by default. This allows the kubelet to pick +# an appropriate value. +{% set hairpin_mode = "" -%} +# The master cannot see Services because it doesn't run kube-proxy, so we don't +# need to make its container bridge promiscuous. We also don't want to set +# the hairpin-veth flag on the master because it increases the chances of +# running into the kernel bug described in #20096. +{% if grains['roles'][0] == 'kubernetes-master' -%} + {% set hairpin_mode = "--hairpin-mode=none" -%} +{% elif pillar['hairpin_mode'] is defined and pillar['hairpin_mode'] in ['promiscuous-bridge', 'hairpin-veth', 'none'] -%} + {% set hairpin_mode = "--hairpin-mode=" + pillar['hairpin_mode'] -%} +{% endif -%} + +{% set babysit_daemons = "" -%} +{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce' ] %} + {% set babysit_daemons = "--babysit-daemons=true" -%} +{% endif -%} + +{% set kubelet_port = "" -%} +{% if pillar['kubelet_port'] is defined -%} + {% set kubelet_port="--port=" + pillar['kubelet_port'] %} +{% endif -%} + +{% set log_level = pillar['log_level'] -%} +{% if pillar['kubelet_test_log_level'] is defined -%} + {% set log_level = pillar['kubelet_test_log_level'] -%} +{% endif -%} + +{% set enable_custom_metrics = "" -%} +{% if pillar['enable_custom_metrics'] is defined -%} + {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %} +{% endif -%} + +{% set node_labels = "" %} +{% if pillar['node_labels'] is defined -%} + {% set node_labels="--node-labels=" + pillar['node_labels'] %} +{% endif -%} + +# test_args has to be kept at the end, so they'll overwrite any prior configuration +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}" diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/init.sls new file mode 100644 index 0000000..a03bf7a --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/init.sls @@ -0,0 +1,85 @@ +{% if pillar.get('is_systemd') %} +{% set environment_file = '/etc/sysconfig/kubelet' %} +{% else %} +{% set environment_file = '/etc/default/kubelet' %} +{% endif %} + +{{ environment_file}}: + file.managed: + - source: salt://kubelet/default + - template: jinja + - user: root + - group: root + - mode: 644 + +/usr/local/bin/kubelet: + file.managed: + - source: salt://kube-bins/kubelet + - user: root + - group: root + - mode: 755 + +# The default here is that this file is blank. If this is the case, the kubelet +# won't be able to parse it as JSON and it will not be able to publish events +# to the apiserver. You'll see a single error line in the kubelet start up file +# about this. +/var/lib/kubelet/kubeconfig: + file.managed: + - source: salt://kubelet/kubeconfig + - user: root + - group: root + - mode: 400 + - makedirs: true + + +{% if pillar.get('is_systemd') %} + +{{ pillar.get('systemd_system_path') }}/kubelet.service: + file.managed: + - source: salt://kubelet/kubelet.service + - user: root + - group: root + +# The service.running block below doesn't work reliably +# Instead we run our script which e.g. does a systemd daemon-reload +# But we keep the service block below, so it can be used by dependencies +# TODO: Fix this +fix-service-kubelet: + cmd.wait: + - name: /opt/kubernetes/helpers/services bounce kubelet + - watch: + - file: /usr/local/bin/kubelet + - file: {{ pillar.get('systemd_system_path') }}/kubelet.service + - file: {{ environment_file }} + - file: /var/lib/kubelet/kubeconfig + +{% else %} + +/etc/init.d/kubelet: + file.managed: + - source: salt://kubelet/initd + - user: root + - group: root + - mode: 755 + +{% endif %} + +kubelet: + service.running: + - enable: True + - watch: + - file: /usr/local/bin/kubelet +{% if pillar.get('is_systemd') %} + - file: {{ pillar.get('systemd_system_path') }}/kubelet.service +{% else %} + - file: /etc/init.d/kubelet +{% endif %} +{% if grains['os_family'] == 'RedHat' %} + - file: /usr/lib/systemd/system/kubelet.service +{% endif %} + - file: {{ environment_file }} + - file: /var/lib/kubelet/kubeconfig +{% if pillar.get('is_systemd') %} + - provider: + - service: systemd +{%- endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/initd b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/initd new file mode 100644 index 0000000..5379c8e --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/initd @@ -0,0 +1,126 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kubelet +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: The Kubernetes node container manager +# Description: +# The Kubernetes container manager maintains docker state against a state file. +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="The Kubernetes container manager" +NAME=kubelet +DAEMON=/usr/local/bin/kubelet +DAEMON_ARGS="" +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=root + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +# +# Function that starts the daemon/service +# +do_start() +{ + # Avoid a potential race at boot time when both monit and init.d start + # the same service + PIDS=$(pidof $DAEMON) + for PID in ${PIDS}; do + kill -9 $PID + done + + # Return + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started + start-stop-daemon --start --quiet --background --no-close \ + --make-pidfile --pidfile $PIDFILE \ + --exec $DAEMON -c $DAEMON_USER --test > /dev/null \ + || return 1 + start-stop-daemon --start --quiet --background --no-close \ + --make-pidfile --pidfile $PIDFILE \ + --exec $DAEMON -c $DAEMON_USER -- \ + $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + # Return + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME + RETVAL="$?" + [ "$RETVAL" = 2 ] && return 2 + # Many daemons don't delete their pidfiles when they exit. + rm -f $PIDFILE + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/kubeconfig b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/kubeconfig new file mode 100644 index 0000000..e69de29 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/kubelet.service b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/kubelet.service new file mode 100644 index 0000000..b266e03 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/kubelet/kubelet.service @@ -0,0 +1,11 @@ +[Unit] +Description=Kubernetes Kubelet Server +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +EnvironmentFile=/etc/sysconfig/kubelet +ExecStart=/usr/local/bin/kubelet "$DAEMON_ARGS" +Restart=always + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/conf b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/conf new file mode 100644 index 0000000..fcd2a67 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/conf @@ -0,0 +1,11 @@ +/var/log/{{ file }}.log { + rotate 5 + copytruncate + missingok + notifempty + compress + maxsize 100M + daily + create 0644 root root +} + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/cron b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/cron new file mode 100755 index 0000000..f8a2a3d --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/cron @@ -0,0 +1,2 @@ +#!/bin/sh +logrotate /etc/logrotate.conf diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/docker-containers b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/docker-containers new file mode 100644 index 0000000..8809851 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/docker-containers @@ -0,0 +1,10 @@ +/var/lib/docker/containers/*/*-json.log { + rotate 5 + copytruncate + missingok + notifempty + compress + maxsize 10M + daily + create 0644 root root +} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/init.sls new file mode 100644 index 0000000..61ca914 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/logrotate/init.sls @@ -0,0 +1,35 @@ +logrotate: + pkg: + - installed + +{% set logrotate_files = ['kube-scheduler', 'kube-proxy', 'kubelet', 'kube-apiserver', 'kube-controller-manager', 'kube-addons', 'docker'] %} +{% for file in logrotate_files %} +/etc/logrotate.d/{{ file }}: + file: + - managed + - source: salt://logrotate/conf + - template: jinja + - user: root + - group: root + - mode: 644 + - context: + file: {{ file }} +{% endfor %} + +/etc/logrotate.d/docker-containers: + file: + - managed + - source: salt://logrotate/docker-containers + - template: jinja + - user: root + - group: root + - mode: 644 + +/etc/cron.hourly/logrotate: + file: + - managed + - source: salt://logrotate/cron + - template: jinja + - user: root + - group: root + - mode: 755 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/init.sls new file mode 100644 index 0000000..2013717 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/init.sls @@ -0,0 +1,64 @@ +nginx: + pkg: + - installed + +/etc/nginx/nginx.conf: + file: + - managed + - source: salt://nginx/nginx.conf + - template: jinja + - user: root + - group: root + - mode: 644 + +/etc/nginx/sites-enabled/default: + file: + - managed + - makedirs: true + - source: salt://nginx/kubernetes-site + - user: root + - group: root + - mode: 644 + +/usr/share/nginx/htpasswd: + file: + - managed + - source: salt://nginx/htpasswd + - user: root + - group: root + - mode: 644 + +{% if grains.cloud is defined and grains.cloud in ['gce'] %} +/etc/kubernetes/manifests/nginx.json: + file: + - managed + - source: salt://nginx/nginx.json + - user: root + - group: root + - mode: 644 + - require: + - file: /etc/nginx/nginx.conf + - file: /etc/nginx/sites-enabled/default + - file: /usr/share/nginx/htpasswd + - cmd: kubernetes-cert + + +#stop legacy nginx_service +stop_nginx-service: + service.dead: + - name: nginx + - enable: None + +{% else %} +nginx-service: + service: + - running + - name: nginx + - watch: + - pkg: nginx + - file: /etc/nginx/nginx.conf + - file: /etc/nginx/sites-enabled/default + - file: /usr/share/nginx/htpasswd + - cmd: kubernetes-cert +{% endif %} + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/kubernetes-site b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/kubernetes-site new file mode 100644 index 0000000..818a487 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/kubernetes-site @@ -0,0 +1,66 @@ +#server { + #listen 80; ## listen for ipv4; this line is default and implied + #listen [::]:80 default_server ipv6only=on; ## listen for ipv6 + +# root /usr/share/nginx/www; +# index index.html index.htm; + + # Make site accessible from http://localhost/ +# server_name localhost; +# location / { +# auth_basic "Restricted"; +# auth_basic_user_file /usr/share/nginx/htpasswd; + + # Proxy settings. +# proxy_pass http://localhost:8080/; +# proxy_connect_timeout 159s; +# proxy_send_timeout 600s; +# proxy_read_timeout 600s; +# proxy_buffer_size 64k; +# proxy_buffers 16 32k; +# proxy_busy_buffers_size 64k; +# proxy_temp_file_write_size 64k; +# } +#} + +# HTTPS server +# +server { + listen 443; + server_name localhost; + + root html; + index index.html index.htm; + + ssl on; + ssl_certificate /srv/kubernetes/server.cert; + ssl_certificate_key /srv/kubernetes/server.key; + + ssl_session_timeout 5m; + + # don't use SSLv3 because of POODLE + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS; + ssl_prefer_server_ciphers on; + + location / { + auth_basic "Restricted"; + auth_basic_user_file /usr/share/nginx/htpasswd; + + # Proxy settings + # disable buffering so that watch works + proxy_buffering off; + proxy_pass http://127.0.0.1:8080/; + proxy_connect_timeout 159s; + proxy_send_timeout 600s; + proxy_read_timeout 600s; + + # Disable retry + proxy_next_upstream off; + + # Support web sockets + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/nginx.conf b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/nginx.conf new file mode 100644 index 0000000..00b1961 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/nginx.conf @@ -0,0 +1,61 @@ +{% if grains['os_family'] == 'RedHat' %} +user nginx; +{% else %} +user www-data; +{% endif %} + +worker_processes 4; +pid /var/run/nginx.pid; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + gzip_disable "msie6"; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/nginx.json b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/nginx.json new file mode 100644 index 0000000..44c41c2 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/nginx/nginx.json @@ -0,0 +1,60 @@ +{ +"apiVersion": "v1", +"kind": "Pod", +"metadata": {"name":"nginx"}, +"spec":{ +"hostNetwork": true, +"containers":[ + { + "name": "nginx", + "image": "gcr.io/google-containers/nginx:v1", + "resources": { + "limits": { + "cpu": "200m" + } + }, + "command": [ + "nginx", + "-g", + "daemon off;" + ], + "ports":[ + { "name": "https", + "containerPort": 443, + "hostPort": 443} + ], + "volumeMounts": [ + { "name": "nginx", + "mountPath": "/etc/nginx", + "readOnly": true}, + { "name": "k8s", + "mountPath": "/srv/kubernetes", + "readOnly": true}, + { "name": "logs", + "mountPath": "/var/log/nginx", + "readOnly": false}, + { "name": "passwd", + "mountPath": "/usr/share/nginx", + "readOnly": true} + ] + } +], +"volumes":[ + { "name": "nginx", + "hostPath": { + "path": "/etc/nginx"} + }, + { "name": "k8s", + "hostPath": { + "path": "/srv/kubernetes"} + }, + { "name": "passwd", + "hostPath": { + "path": "/usr/share/nginx"} + }, + { "name": "logs", + "hostPath": { + "path": "/var/logs/nginx"} + } +] +}} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/ntp/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/ntp/init.sls new file mode 100644 index 0000000..fa8c8a7 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/ntp/init.sls @@ -0,0 +1,11 @@ +ntp: + pkg: + - installed + +ntp-service: + service: + - running + - name: ntp + - watch: + - pkg: ntp + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/opencontrail-networking-master/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/opencontrail-networking-master/init.sls new file mode 100644 index 0000000..78290bb --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/opencontrail-networking-master/init.sls @@ -0,0 +1,15 @@ +opencontrail-networking-master: + cmd.script: + - unless: test -f /var/log/contrail/provision_master.log + - env: + - 'OPENCONTRAIL_TAG': '{{ pillar.get('opencontrail_tag') }}' + - 'OPENCONTRAIL_KUBERNETES_TAG': '{{ pillar.get('opencontrail_kubernetes_tag') }}' + - 'OPENCONTRAIL_PUBLIC_SUBNET': '{{ pillar.get('opencontrail_public_subnet') }}' + - 'SERVICE_CLUSTER_IP_RANGE': '{{ pillar.get('service_cluster_ip_range') }}' + - source: https://raw.githubusercontent.com/juniper/contrail-kubernetes/{{ pillar.get('opencontrail_kubernetes_tag') }}/cluster/provision_master.sh + - source_hash: https://raw.githubusercontent.com/juniper/contrail-kubernetes/{{ pillar.get('opencontrail_kubernetes_tag') }}/cluster/manifests.hash + - cwd: / + - user: root + - group: root + - mode: 755 + - shell: /bin/bash diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/opencontrail-networking-minion/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/opencontrail-networking-minion/init.sls new file mode 100644 index 0000000..7cb68de --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/opencontrail-networking-minion/init.sls @@ -0,0 +1,15 @@ +opencontrail-networking-minion: + cmd.script: + - unless: test -f /var/log/contrail/provision_minion.log + - env: + - 'OPENCONTRAIL_TAG': '{{ pillar.get('opencontrail_tag') }}' + - 'OPENCONTRAIL_KUBERNETES_TAG': '{{ pillar.get('opencontrail_kubernetes_tag') }}' + - 'OPENCONTRAIL_PUBLIC_SUBNET': '{{ pillar.get('opencontrail_public_subnet') }}' + - 'SERVICE_CLUSTER_IP_RANGE': '{{ pillar.get('service_cluster_ip_range') }}' + - source: https://raw.githubusercontent.com/juniper/contrail-kubernetes/{{ pillar.get('opencontrail_kubernetes_tag') }}/cluster/provision_minion.sh + - source_hash: https://raw.githubusercontent.com/juniper/contrail-kubernetes/{{ pillar.get('opencontrail_kubernetes_tag') }}/cluster/manifests.hash + - cwd: / + - user: root + - group: root + - mode: 755 + - shell: /bin/bash diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/init.sls new file mode 100644 index 0000000..3898001 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/init.sls @@ -0,0 +1,24 @@ +/opt/kubernetes/helpers: + file.directory: + - user: root + - group: root + - makedirs: True + - dir_mode: 755 + +{% if pillar.get('is_systemd') %} +/opt/kubernetes/helpers/services: + file.managed: + - source: salt://salt-helpers/services + - user: root + - group: root + - mode: 755 +{% endif %} + +{% if grains.get('os_family', '') == 'Debian' -%} +/opt/kubernetes/helpers/pkg: + file.managed: + - source: salt://salt-helpers/pkg-apt + - user: root + - group: root + - mode: 755 +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/pkg-apt b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/pkg-apt new file mode 100644 index 0000000..803d301 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/pkg-apt @@ -0,0 +1,70 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper script that installs a package, wrapping it with a policy that +# means we won't try to start services. +set -o errexit +set -o nounset +set -o pipefail + +ACTION=${1} +NAME=${2} +VERSION=${3} +SRC=${4} + +if [[ -z "${ACTION}" || -z "${NAME}" || -z "${VERSION}" || -z "${SRC}" ]]; then + echo "Syntax: ${0} " + exit 1 +fi + +old_policy="" + +function install_no_start { + # Query the existing installed version, assuming that an error means package not found + existing=`dpkg-query -W -f='${Version}' ${NAME} 2>/dev/null || echo ""` + if [[ -n "${existing}" ]]; then + if [[ "${existing}" == "${VERSION}" ]]; then + return + fi + echo "Different version of package ${NAME} installed: ${VERSION} vs ${existing}" + fi + + if [[ -e "/usr/sbin/policy-rc.d" ]]; then + tmpfile=`mktemp` + mv /usr/sbin/policy-rc.d ${tmpfile} + old_policy=${tmpfile} + fi + trap cleanup EXIT + echo -e '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d + chmod 755 /usr/sbin/policy-rc.d + + echo "Installing package ${NAME} from ${SRC}" + dpkg --install ${SRC} +} + +function cleanup { + rm -f /usr/sbin/policy-rc.d + if [[ -n "${old_policy}" ]]; then + mv ${old_policy} /usr/sbin/policy-rc.d + fi +} + +if [[ "${ACTION}" == "install-no-start" ]]; then + install_no_start +else + echo "Unknown action: ${ACTION}" + exit 1 +fi diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/services b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/services new file mode 100644 index 0000000..bc8db58 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/salt-helpers/services @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +ACTION=${1} +SERVICE=${2} + +if [[ -z "${ACTION}" || -z "${SERVICE}" ]]; then + echo "Syntax: ${0} " + exit 1 +fi + + +function reload_state() { + systemctl daemon-reload +} + +function start_service() { + systemctl start ${SERVICE} +} + +function stop_service() { + systemctl stop ${SERVICE} +} + +function enable_service() { + systemctl enable ${SERVICE} +} + +function disable_service() { + systemctl disable ${SERVICE} +} + +function restart_service() { + systemctl restart ${SERVICE} +} + +if [[ "${ACTION}" == "up" ]]; then + reload_state + enable_service + start_service +elif [[ "${ACTION}" == "bounce" ]]; then + reload_state + enable_service + restart_service +elif [[ "${ACTION}" == "down" ]]; then + reload_state + disable_service + stop_service +elif [[ "${ACTION}" == "enable" ]]; then + reload_state + enable_service +else + echo "Unknown action: ${ACTION}" + exit 1 +fi diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/if-down b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/if-down new file mode 100644 index 0000000..9da2865 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/if-down @@ -0,0 +1,16 @@ +#!/bin/bash + +[ "$IFACE" == "eth0" ] || exit 0 + +{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %} + {% set network_ipaddr = None %} + {% if salt['network.ip_addrs']('eth0') is defined %} + {% set network_ipaddr = salt['network.ip_addrs']('eth0')[0] %} + {% elif salt['network.ip_addrs']('local') is defined %} + {% set network_ipaddr = salt['network.ip_addrs']('local')[0] %} + {% endif %} + {% if network_ipaddr is defined and ip_addrs[0] != network_ipaddr %} + {% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %} + route del -net {{ cidr }} + {% endif %} +{% endfor %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/if-up b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/if-up new file mode 100644 index 0000000..3661407 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/if-up @@ -0,0 +1,16 @@ +#!/bin/bash + +[ "$IFACE" == "eth0" ] || exit 0 + +{% for host, ip_addrs in salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', 'grain').items() %} + {% set network_ipaddr = None %} + {% if salt['network.ip_addrs']('eth0') %} + {% set network_ipaddr = salt['network.ip_addrs']('eth0')[0] %} + {% elif salt['network.ip_addrs']('local') %} + {% set network_ipaddr = salt['network.ip_addrs']('local').first %} + {% endif %} + {% if network_ipaddr and ip_addrs[0] != network_ipaddr %} + {% set cidr = salt['mine.get'](host, 'grains.items')[host]['cbr-cidr'] %} + route add -net {{ cidr }} gw {{ ip_addrs[0] }} + {% endif %} +{% endfor %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/init.sls new file mode 100644 index 0000000..3a01c55 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/init.sls @@ -0,0 +1,30 @@ +# Add static routes to every minion to enable pods in the 10.244.x.x range to +# reach each other. This is suboptimal, but necessary to let every pod have +# its IP and have pods between minions be able to talk with each other. +# This will be obsolete when we figure out the right way to make this work. + +/etc/network/if-up.d/static-routes: + file.managed: + - source: salt://static-routes/if-up + - template: jinja + - user: root + - group: root + - mode: 755 + +/etc/network/if-down.d/static-routes: + file.managed: + - source: salt://static-routes/if-down + - template: jinja + - user: root + - group: root + - mode: 755 + +refresh-routes: + cmd.wait_script: + - source: salt://static-routes/refresh + - cwd: /etc/network/ + - user: root + - group: root + - watch: + - file: /etc/network/if-up.d/static-routes + - file: /etc/network/if-down.d/static-routes diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/refresh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/refresh new file mode 100644 index 0000000..e988269 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/static-routes/refresh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Fake an ifup/ifdown event +export IFACE=eth0 + +if-down.d/static-routes || true +if-up.d/static-routes || true diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/docker-checker.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/docker-checker.sh new file mode 100755 index 0000000..7e615c5 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/docker-checker.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is intended to start the docker and then loop until +# it detects a failure. It then exits, and supervisord restarts it +# which in turn restarts docker. + +/etc/init.d/docker stop +# Make sure docker gracefully terminated before start again +while pidof docker > /dev/null; do + echo "waiting clean shutdown" + sleep 10 +done + +# cleanup docker network checkpoint to avoid running into known issue +# of docker (https://github.com/docker/docker/issues/18283) +rm -rf /var/lib/docker/network + +/etc/init.d/docker start + +echo "waiting 30s for startup" +sleep 30 + +while true; do + if ! sudo timeout 10 docker version > /dev/null; then + echo "Docker failed!" + exit 2 + fi + sleep 10 +done + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/docker.conf b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/docker.conf new file mode 100644 index 0000000..3fd61f3 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/docker.conf @@ -0,0 +1,6 @@ +[program:docker] +command=/usr/sbin/docker-checker.sh +stderr_logfile=/var/log/supervisor/docker-stderr.log +stdout_logfile=/var/log/supervisor/docker-stdout.log +autorestart=true +startretries=1000000 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/init.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/init.sls new file mode 100644 index 0000000..fb2b387 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/init.sls @@ -0,0 +1,102 @@ +{% if not pillar.get('is_systemd') %} + +supervisor: + pkg: + - installed + +monit: + pkg: + - purged + +/etc/supervisor/conf.d/docker.conf: + file: + - managed + - source: salt://supervisor/docker.conf + - user: root + - group: root + - mode: 644 + - makedirs: True + - require_in: + - pkg: supervisor + - require: + - file: /usr/sbin/docker-checker.sh + +/usr/sbin/docker-checker.sh: + file: + - managed + - source: salt://supervisor/docker-checker.sh + - user: root + - group: root + - mode: 755 + - makedirs: True + +/etc/supervisor/conf.d/kubelet.conf: + file: + - managed + - source: salt://supervisor/kubelet.conf + - user: root + - group: root + - mode: 644 + - makedirs: True + - require_in: + - pkg: supervisor + - require: + - file: /usr/sbin/kubelet-checker.sh + +/usr/sbin/kubelet-checker.sh: + file: + - managed + - source: salt://supervisor/kubelet-checker.sh + - template: jinja + - user: root + - group: root + - mode: 755 + - makedirs: True + +{% if grains['roles'][0] == 'kubernetes-master' -%} +/etc/supervisor/conf.d/kube-addons.conf: + file: + - managed + - source: salt://supervisor/kube-addons.conf + - user: root + - group: root + - mode: 644 + - makedirs: True + - require_in: + - pkg: supervisor + - require: + - file: /usr/sbin/kube-addons-checker.sh + +/usr/sbin/kube-addons-checker.sh: + file: + - managed + - source: salt://supervisor/kube-addons-checker.sh + - user: root + - group: root + - mode: 755 + - makedirs: True +{% endif %} + +/etc/supervisor/supervisor_watcher.sh: + file.managed: + - source: salt://supervisor/supervisor_watcher.sh + - user: root + - group: root + - mode: 755 + - makedirs: True + +crontab -l | { cat; echo "* * * * * /etc/supervisor/supervisor_watcher.sh 2>&1 | logger"; } | crontab -: + cmd.run: + - unless: crontab -l | grep "* * * * * /etc/supervisor/supervisor_watcher.sh 2>&1 | logger" + +supervisor-service: + service: + - running + - name: supervisor + - watch: + - pkg: supervisor + - file: /etc/supervisor/conf.d/* + - require: + - pkg: supervisor + +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kube-addons-checker.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kube-addons-checker.sh new file mode 100644 index 0000000..d974973 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kube-addons-checker.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is intended to start the kube-addons and then loop until +# it detects a failure. It then exits, and supervisord restarts it +# which in turn restarts the kube-addons. + +/etc/init.d/kube-addons stop +/etc/init.d/kube-addons start + +echo "waiting a minute for startup" +sleep 60 + +while true; do + if ! /etc/init.d/kube-addons status > /dev/null; then + echo "kube-addons failed!" + exit 2 + fi + sleep 10 +done + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kube-addons.conf b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kube-addons.conf new file mode 100644 index 0000000..db977ae --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kube-addons.conf @@ -0,0 +1,6 @@ +[program:kube-addons] +command=/usr/sbin/kube-addons-checker.sh +stderr_logfile=/var/log/supervisor/kube-addons-stderr.log +stdout_logfile=/var/log/supervisor/kube-addons-stdout.log +autorestart=true +startretries=1000000 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kubelet-checker.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kubelet-checker.sh new file mode 100755 index 0000000..41fb061 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kubelet-checker.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is intended to start the kubelet and then loop until +# it detects a failure. It then exits, and supervisord restarts it +# which in turn restarts the kubelet. + +{% set kubelet_port = "10250" -%} +{% if pillar['kubelet_port'] is defined -%} + {% set kubelet_port = pillar['kubelet_port'] -%} +{% endif -%} + +/etc/init.d/kubelet stop +/etc/init.d/kubelet start + +echo "waiting a minute for startup" +sleep 60 + +max_seconds=10 + +while true; do + if ! curl --insecure -m ${max_seconds} -f -s https://127.0.0.1:{{kubelet_port}}/healthz > /dev/null; then + echo "kubelet failed!" + curl --insecure -s http://127.0.0.1:{{kubelet_port}}/healthz + exit 2 + fi + sleep 10 +done + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kubelet.conf b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kubelet.conf new file mode 100644 index 0000000..25f8274 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/kubelet.conf @@ -0,0 +1,6 @@ +[program:kubelet] +command=/usr/sbin/kubelet-checker.sh +stderr_logfile=/var/log/supervisor/kubelet-stderr.log +stdout_logfile=/var/log/supervisor/kubelet-stdout.log +autorestart=true +startretries=1000000 diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/supervisor_watcher.sh b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/supervisor_watcher.sh new file mode 100644 index 0000000..da21452 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/supervisor/supervisor_watcher.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is invoked by crond every minute to check if supervisord is +# up and oom protected. If down it restarts supervisord; otherwise, it exits +# after applying oom_score_adj + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +if ! /etc/init.d/supervisor status > /dev/null; then + service supervisor start + sleep 10 +fi + +# Apply oom_score_adj: -901 to processes +pids=$(cat /var/run/supervisord.pid) +for pid in "${pids}"; do + echo -901 > /proc/$pid/oom_score_adj +done + + diff --git a/hack/kubernetes-clearwater/kubernetes/saltbase/salt/top.sls b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/top.sls new file mode 100644 index 0000000..0f1db45 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/saltbase/salt/top.sls @@ -0,0 +1,82 @@ +base: + '*': + - base + - debian-auto-upgrades + - salt-helpers +{% if grains.get('cloud') == 'aws' %} + - ntp +{% endif %} +{% if pillar.get('e2e_storage_test_environment', '').lower() == 'true' %} + - e2e +{% endif %} + + 'roles:kubernetes-pool': + - match: grain + - docker +{% if pillar.get('network_provider', '').lower() == 'flannel' %} + - flannel +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + - cni +{% endif %} + - helpers + - cadvisor + - kube-client-tools + - kube-node-unpacker + - kubelet +{% if pillar.get('network_provider', '').lower() == 'opencontrail' %} + - opencontrail-networking-minion +{% else %} + - kube-proxy +{% endif %} +{% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar['logging_destination'] is defined %} + {% if pillar['logging_destination'] == 'elasticsearch' %} + - fluentd-es + {% elif pillar['logging_destination'] == 'gcp' %} + - fluentd-gcp + {% endif %} +{% endif %} +{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %} + - kube-registry-proxy +{% endif %} + - logrotate + - supervisor + + 'roles:kubernetes-master': + - match: grain + - generate-cert + - etcd +{% if pillar.get('network_provider', '').lower() == 'flannel' %} + - flannel-server + - flannel +{% elif pillar.get('network_provider', '').lower() == 'kubenet' %} + - cni +{% endif %} + - kube-apiserver + - kube-controller-manager + - kube-scheduler + - supervisor +{% if grains['cloud'] is defined and not grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere'] %} + - nginx +{% endif %} + - cadvisor + - kube-client-tools + - kube-master-addons + - kube-admission-controls +{% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar['logging_destination'] is defined %} + {% if pillar['logging_destination'] == 'elasticsearch' %} + - fluentd-es + {% elif pillar['logging_destination'] == 'gcp' %} + - fluentd-gcp + {% endif %} +{% endif %} +{% if grains['cloud'] is defined and grains['cloud'] != 'vagrant' %} + - logrotate +{% endif %} + - kube-addons +{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere' ] %} + - docker + - kubelet +{% endif %} +{% if pillar.get('network_provider', '').lower() == 'opencontrail' %} + - opencontrail-networking-master +{% endif %} diff --git a/hack/kubernetes-clearwater/kubernetes/skydns-issue.md b/hack/kubernetes-clearwater/kubernetes/skydns-issue.md new file mode 100755 index 0000000..8f5d404 --- /dev/null +++ b/hack/kubernetes-clearwater/kubernetes/skydns-issue.md @@ -0,0 +1,47 @@ + +To 1.3.6 + + [vagrant@localhost ~]$ kubectl --namespace=kube-system get svc,rc + + [vagrant@localhost ~]$ kubectl create -f kubernetes/1.3.6/manifests/addons/dns/skydns-rc.yaml + replicationcontroller "kube-dns-v20" created + + [vagrant@localhost ~]$ kubectl create -f kubernetes/1.3.6/manifests/addons/dns/skydns-svc.yaml + service "kube-dns" created + + [vagrant@localhost ~]$ kubectl --namespace=kube-system get pods,ep,svc,rc + NAME READY STATUS RESTARTS AGE + kube-dns-v20-qci0l 2/3 CrashLoopBackOff 4 2m + NAME ENDPOINTS AGE + kube-dns 2m + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + kube-dns 10.123.240.10 53/UDP,53/TCP 2m + NAME DESIRED CURRENT AGE + kube-dns-v20 1 1 9m + + [vagrant@localhost ~]$ kubectl --namespace=kube-system logs kube-dns-v20-qci0l -c kubedns + F1115 11:49:47.951866 1 server.go:55] Failed to create a kubernetes client: invalid configuration: no configuration has been provided + + [vagrant@localhost ~]$ kubectl --namespace=kube-system logs kube-dns-v20-qci0l -c dnsmasq + dnsmasq[1]: started, version 2.76 cachesize 1000 + dnsmasq[1]: compile time options: IPv6 GNU-getopt no-DBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP no-conntrack ipset auth no-DNSSEC loop-detect inotify + dnsmasq[1]: using nameserver 127.0.0.1#10053 + dnsmasq[1]: failed to load names from /etc/hosts: Permission denied + + [vagrant@localhost ~]$ kubectl --namespace=kube-system logs kube-dns-v20-qci0l -c healthz + 2016/11/15 11:47:54 Healthz probe on /healthz-dnsmasq error: Result of last exec: nslookup: can't resolve 'kubernetes.default.svc.cluster.local' + , at 2016-11-15 11:47:53.967915568 +0000 UTC, error exit status 1 + 2016/11/15 11:48:04 Healthz probe on /healthz-dnsmasq error: Result of last exec: nslookup: can't resolve 'kubernetes.default.svc.cluster.local' + , at 2016-11-15 11:48:03.966702303 +0000 UTC, error exit status 1 + 2016/11/15 11:48:14 Healthz probe on /healthz-dnsmasq error: Result of last exec: nslookup: can't resolve 'kubernetes.default.svc.cluster.local' + , at 2016-11-15 11:48:13.96200954 +0000 UTC, error exit status 1 + 2016/11/15 11:48:24 Healthz probe on /healthz-dnsmasq error: Result of last exec: nslookup: can't resolve 'kubernetes.default.svc.cluster.local' + , at 2016-11-15 11:48:23.971159243 +0000 UTC, error exit status 1 + 2016/11/15 11:48:34 Healthz probe on /healthz-dnsmasq error: Result of last exec: nslookup: can't resolve 'kubernetes.default.svc.cluster.local' + , at 2016-11-15 11:48:34.111545116 +0000 UTC, error exit status 1 + 2016/11/15 11:50:04 Healthz probe on /healthz-dnsmasq error: Result of last exec: nslookup: can't resolve 'kubernetes.default.svc.cluster.local' + , at 2016-11-15 11:50:03.961031994 +0000 UTC, error exit status 1 + + [vagrant@localhost ~]$ kubectl --namespace=kube-system delete svc/kube-dns rc/kube-dns-v20 + service "kube-dns" deleted + replicationcontroller "kube-dns-v20" deleted diff --git a/hack/kubernetes-clearwater/lc-tlscert/lc-tlscert.go b/hack/kubernetes-clearwater/lc-tlscert/lc-tlscert.go new file mode 100644 index 0000000..50ac047 --- /dev/null +++ b/hack/kubernetes-clearwater/lc-tlscert/lc-tlscert.go @@ -0,0 +1,204 @@ +/* +* Copyright 2014-2015 Jason Woods. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* Derived from Golang src/pkg/crypto/tls/generate_cert.go +* Copyright 2009 The Go Authors. All rights reserved. +* Use of this source code is governed by a BSD-style +* license that can be found in the LICENSE file. + */ + +package main + +import ( + "bufio" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "strconv" + "time" +) + +var input *bufio.Reader + +func init() { + input = bufio.NewReader(os.Stdin) +} + +func readString(prompt string) string { + fmt.Printf("%s: ", prompt) + + var line []byte + for { + data, prefix, _ := input.ReadLine() + line = append(line, data...) + if !prefix { + break + } + } + + return string(line) +} + +func readNumber(prompt string) (num int64) { + var err error + for { + if num, err = strconv.ParseInt(readString(prompt), 0, 64); err != nil { + fmt.Println("Please enter a valid numerical value") + continue + } + break + } + return +} + +func anyKey() { + input.ReadRune() +} + +func main() { + var err error + + template := x509.Certificate{ + Subject: pkix.Name{ + Organization: []string{"Log Courier"}, + }, + NotBefore: time.Now(), + + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + + IsCA: true, + } + + fmt.Println("Specify the Common Name for the certificate. The common name") + fmt.Println("can be anything, but is usually set to the server's primary") + fmt.Println("DNS name. Even if you plan to connect via IP address you") + fmt.Println("should specify the DNS name here.") + fmt.Println() + + template.Subject.CommonName = readString("Common name") + fmt.Println() + + fmt.Println("The next step is to add any additional DNS names and IP") + fmt.Println("addresses that clients may use to connect to the server. If") + fmt.Println("you plan to connect to the server via IP address and not DNS") + fmt.Println("then you must specify those IP addresses here.") + fmt.Println("When you are finished, just press enter.") + fmt.Println() + + var cnt = 0 + var val string + for { + cnt++ + + if val = readString(fmt.Sprintf("DNS or IP address %d", cnt)); val == "" { + break + } + + if ip := net.ParseIP(val); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, val) + } + } + + fmt.Println() + + fmt.Println("How long should the certificate be valid for? A year (365") + fmt.Println("days) is usual but requires the certificate to be regenerated") + fmt.Println("within a year or the certificate will cease working.") + fmt.Println() + + template.NotAfter = template.NotBefore.Add(time.Duration(readNumber("Number of days")) * time.Hour * 24) + + fmt.Println("Common name:", template.Subject.CommonName) + fmt.Println("DNS SANs:") + if len(template.DNSNames) == 0 { + fmt.Println(" None") + } else { + for _, e := range template.DNSNames { + fmt.Println(" ", e) + } + } + fmt.Println("IP SANs:") + if len(template.IPAddresses) == 0 { + fmt.Println(" None") + } else { + for _, e := range template.IPAddresses { + fmt.Println(" ", e) + } + } + fmt.Println() + + fmt.Println("The certificate can now be generated") + fmt.Println("Press any key to begin generating the self-signed certificate.") + anyKey() + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + fmt.Println("Failed to generate private key:", err) + os.Exit(1) + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + template.SerialNumber, err = rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + fmt.Println("Failed to generate serial number:", err) + os.Exit(1) + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + fmt.Println("Failed to create certificate:", err) + os.Exit(1) + } + + certOut, err := os.Create("selfsigned.crt") + if err != nil { + fmt.Println("Failed to open selfsigned.pem for writing:", err) + os.Exit(1) + } + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + certOut.Close() + + keyOut, err := os.OpenFile("selfsigned.key", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + fmt.Println("failed to open selfsigned.key for writing:", err) + os.Exit(1) + } + pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) + keyOut.Close() + + fmt.Println("Successfully generated certificate") + fmt.Println(" Certificate: selfsigned.crt") + fmt.Println(" Private Key: selfsigned.key") + fmt.Println() + fmt.Println("Copy and paste the following into your Log Courier") + fmt.Println("configuration, adjusting paths as necessary:") + fmt.Println(" \"transport\": \"tls\",") + fmt.Println(" \"ssl ca\": \"path/to/selfsigned.crt\",") + fmt.Println() + fmt.Println("Copy and paste the following into your LogStash configuration, ") + fmt.Println("adjusting paths as necessary:") + fmt.Println(" ssl_certificate => \"path/to/selfsigned.crt\",") + fmt.Println(" ssl_key => \"path/to/selfsigned.key\",") +} diff --git a/hack/kubernetes-clearwater/registry-installation.md b/hack/kubernetes-clearwater/registry-installation.md new file mode 100755 index 0000000..8215ab1 --- /dev/null +++ b/hack/kubernetes-clearwater/registry-installation.md @@ -0,0 +1,437 @@ +Instruction +------------ + +Image pull or load + + [vagrant@localhost ~]$ docker pull docker.io/registry:2.4.1 + Trying to pull repository docker.io/library/registry ... + 2.4.1: Pulling from docker.io/library/registry + 5c90d4a2d1a8: Pull complete + fb8b2153aae6: Pull complete + f719459a7672: Pull complete + fa42982c9892: Pull complete + Digest: sha256:504b44c0ca43f9243ffa6feaf3934dd57895aece36b87bc25713588cdad3dd10 + Status: Downloaded newer image for docker.io/registry:2.4.1 + + [vagrant@localhost ~]$ docker load --input=docker.io%2Fregistry%3A2.4.1.tar + +### Install as a `kubernetes` POD + +Create kubernetes namespace + + [vagrant@localhost ~]$ vi kubernetes/registry/2.4.1/stackdocker-namespace.yaml + apiVersion: v1 + kind: Namespace + metadata: + name: stackdocker + +* Registry account + +Create `htpasswd` + + [vagrant@localhost ~]$ docker run --entrypoint=htpasswd --rm registry:2.4.1 -Bbn admin admin123 > htpasswd + + [vagrant@localhost ~]$ mv htpasswd kubernetes/registry/2.4.1/htpasswd + +Kubernetes `Secret` + + [vagrant@localhost ~]$ echo "admin:$2y$05$PjSkj20tpMSWabyzrpTMlug/sUgwXA7VZNmpl0w/xloLoO6drBINC" | base64 -w 0 + YWRtaW46JDJ5JDA1JEpCNUVUdUxrODFoMmhIZTlFSkZkZk9peDh2R005M2o0c1k1QmZrRkxqR0FjMmZuZjByWDh1Cgo= + + [vagrant@localhost ~]$ vi kubernetes/registry/2.4.1/htpasswd-secret.yaml + apiVersion: v1 + kind: Secret + metadata: + name: registry-htpasswd + namespace: stackdocker + data: + htpasswd: ** + type: Opaque + +* Self-signed Cert + +Reference + + https://github.com/stackdocker/log-courier/blob/master/lc-tlscert/lc-tlscert.go + +Build in `Golang` + + [vagrant@localhost ~]$ go get github.com/stackdocker/log-courier/lc-tlscert + +Generate + + [vagrant@localhost ~]$ lc-tlscert + Specify the Common Name for the certificate. The common name + can be anything, but is usually set to the server's primary + DNS name. Even if you plan to connect via IP address you + should specify the DNS name here. + + Common name: 10.64.33.81:5000 + + The next step is to add any additional DNS names and IP + addresses that clients may use to connect to the server. If + you plan to connect to the server via IP address and not DNS + then you must specify those IP addresses here. + When you are finished, just press enter. + + DNS or IP address 1: 10.64.33.81 + DNS or IP address 2: 10.123.240.35 + DNS or IP address 3: registry.default.svc.cluster.local + DNS or IP address 4: registry.stackdocker.svc.cluster.local + DNS or IP address 5: + + + How long should the certificate be valid for? A year (365 + days) is usual but requires the certificate to be regenerated + within a year or the certificate will cease working. + + Number of days: 1000 + Common name: 10.64.33.81:5000 + DNS SANs: + registry.default.svc.cluster.local + registry.stackdocker.svc.cluster.local + IP SANs: + 10.64.33.81 + 10.123.240.35 + + The certificate can now be generated + Press any key to begin generating the self-signed certificate. + + Successfully generated certificate + Certificate: selfsigned.crt + Private Key: selfsigned.key + + Copy and paste the following into your Log Courier + configuration, adjusting paths as necessary: + "transport": "tls", + "ssl ca": "path/to/selfsigned.crt", + + Copy and paste the following into your LogStash configuration, + adjusting paths as necessary: + ssl_certificate => "path/to/selfsigned.crt", + ssl_key => "path/to/selfsigned.key", + +Add in `docker-engine` option, e.g. *worker nodes* and any of this *registry clients* + + [vagrant@localhost ~]$ sudo mkdir -p /etc/docker/certs.d/10.64.33.81:5000 + + [vagrant@localhost ~]$ sudo cp selfsigned.crt /etc/docker/certs.d/10.64.33.81\:5000/ca.crt + + [vagrant@localhost ~]$ sudo cp selfsigned.key /etc/docker/certs.d/10.64.33.81\:5000/tls.key + + [vagrant@localhost 2.4.1]$ sudo vi /etc/sysconfig/docker + # Modify these options if you want to change the way the docker daemon runs + OPTIONS='--selinux-enabled --log-driver=journald --insecure-registry=192.168.0.0/16 --insecure-registry=172.16.0.0/12 --insecure-registry=10.0.0.0/8' + + [vagrant@localhost 2.4.1]$ sudo systemctl restart docker.service + +Save for clients used laterly + + [vagrant@localhost ~]$ mv selfsigned.crt kubernetes/registry/2.4.1/tls.crt + + [vagrant@localhost ~]$ mv selfsigned.key kubernetes/registry/2.4.1/tls.key + +kubernetes `secret` + + [vagrant@localhost ~]$ base64 -w 0 kubernetes/registry/2.4.1/tls.crt + + [vagrant@localhost ~]$ base64 -w 0 kubernetes/registry/2.4.1/tls.key + + [vagrant@localhost ~]$ vi kubernetes/registry/2.4.1/tls-secret.yaml + apiVersion: v1 + kind: Secret + metadata: + name: registry-tls + namespace: stackdocker + type: kubernetes.io/tls + data: + tls.crt: ** + tls.key: ** + +* Configuration + +Reference + + https://github.com/docker/distribution/blob/master/docs/configuration.md#registry-configuration-reference + + [vagrant@localhost ~]$ vi kubernetes/registry/2.4.1/config.yml + version: 0.1 + log: + fields: + service: registry + storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry + http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + health: + storagedriver: + enabled: true + interval: 30s + threshold: 3 + auth: + htpasswd: + realm: basic-realm + path: /auth/htpasswd + + [vagrant@localhost ~]$ vi kubernetes/registry/2.4.1/config-configmap.yaml + apiVersion: v1 + kind: ConfigMap + metadata: + name: registry-config + namespace: stackdocker + data: + config.yml: | + version: 0.1 + log: + fields: + service: registry + storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry + http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + health: + storagedriver: + enabled: true + interval: 30s + threshold: 3 + auth: + htpasswd: + realm: basic-realm + path: /auth/htpasswd + +* Kubernetes manifests + +deployment + + [vagrant@localhost ~]$ vi kubernetes/registry/2.4.1/registry-deployment.yaml + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: registry + namespace: stackdocker + spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + stackdocker-app: docker-distribution + template: + metadata: + labels: + stackdocker-app: docker-distribution + spec: + containers: + - env: + - name: DOCKER_REGISTRY_CONFIG + value: /etc/docker/registry/config.yml + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /certs/tls.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /certs/tls.key + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + image: docker.io/registry:2.4.1 + name: registry + ports: + - containerPort: 5000 + hostIP: 10.64.33.81 + hostPort: 5000 + securityContext: + privileged: true + volumeMounts: + - name: basic-realm + mountPath: /auth + readOnly: true + - name: selfsigned-tls + mountPath: /certs + readOnly: true + - name: config + mountPath: /etc/docker/registry + readOnly: true + - name: repo + mountPath: /var/lib/registry + volumes: + - name: basic-realm + secret: + secretName: registry-htpasswd + items: + - key: htpasswd + path: htpasswd + - name: selfsigned-tls + secret: + secretName: registry-tls + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + - name: config + configMap: + name: registry-config + items: + - key: config.yml + path: config.yml + - name: repo + hostPath: /var/lib/registry + +Service + + [vagrant@localhost ~]$ vi kubernetes/registry/2.4.1/registry-service.yaml + apiVersion: v1 + kind: Service + metadata: + name: registry + namespace: stackdocker + spec: + clusterIP: 10.123.240.35 + ports: + - port: 5000 + selector: + stackdocker-app: docker-distribution + +* Start + + [vagrant@localhost ~]$ kubectl create -f kubernetes/registry/2.4.1/config-configmap.yaml + configmap "registry-config" created + + [vagrant@localhost ~]$ kubectl create -f kubernetes/registry/2.4.1/htpasswd-secret.yaml + secret "registry-htpasswd" created + + [vagrant@localhost ~]$ kubectl create -f kubernetes/registry/2.4.1/tls-secret.yaml + secret "registry-tls" created + + [vagrant@localhost ~]$ kubectl create -f kubernetes/registry/2.4.1/registry-deployment.yaml + deployment "registry" created + + [vagrant@localhost ~]$ kubectl create -f kubernetes/registry/2.4.1/registry-service.yaml + service "registry" created + +* Validate + + [vagrant@localhost ~]$ kubectl --namespace=stackdocker get pods,ep,svc -o wide + NAME READY STATUS RESTARTS AGE IP NODE + registry-2499285598-mp80z 1/1 Running 0 1m 10.121.24.5 10.64.33.81 + NAME ENDPOINTS AGE + registry 10.121.24.5:5000 40s + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR + registry 10.123.240.35 5000/TCP 40s stackdocker-app=docker-distribution + + [vagrant@localhost ~]$ kubectl --namespace=stackdocker logs registry-2499285598-mp80z + time="2016-11-14T21:18:08Z" level=warning msg="Ignoring unrecognized environment variable REGISTRY_PORT" + time="2016-11-14T21:18:08Z" level=warning msg="Ignoring unrecognized environment variable REGISTRY_PORT_5000_TCP" + time="2016-11-14T21:18:08Z" level=warning msg="Ignoring unrecognized environment variable REGISTRY_PORT_5000_TCP_ADDR" + time="2016-11-14T21:18:08Z" level=warning msg="Ignoring unrecognized environment variable REGISTRY_PORT_5000_TCP_PORT" + time="2016-11-14T21:18:08Z" level=warning msg="Ignoring unrecognized environment variable REGISTRY_PORT_5000_TCP_PROTO" + time="2016-11-14T21:18:08Z" level=warning msg="Ignoring unrecognized environment variable REGISTRY_SERVICE_HOST" + time="2016-11-14T21:18:08Z" level=warning msg="Ignoring unrecognized environment variable REGISTRY_SERVICE_PORT" + time="2016-11-14T21:18:08Z" level=warning msg="No HTTP secret provided - generated random secret. This may cause problems with uploads if multiple registries are behind a load-balancer. To provide a shared secret, fill in http.secret in the configuration file or set the REGISTRY_HTTP_SECRET environment variable." go.version=go1.6.2 instance.id=bd30687b-c884-4f01-8164-1b40abbc6bee version=v2.4.1 + time="2016-11-14T21:18:08Z" level=info msg="redis not configured" go.version=go1.6.2 instance.id=bd30687b-c884-4f01-8164-1b40abbc6bee version=v2.4.1 + time="2016-11-14T21:18:08Z" level=info msg="Starting upload purge in 52m0s" go.version=go1.6.2 instance.id=bd30687b-c884-4f01-8164-1b40abbc6bee version=v2.4.1 + time="2016-11-14T21:18:08Z" level=info msg="using inmemory blob descriptor cache" go.version=go1.6.2 instance.id=bd30687b-c884-4f01-8164-1b40abbc6bee version=v2.4.1 + time="2016-11-14T21:18:08Z" level=info msg="listening on [::]:5000, tls" go.version=go1.6.2 instance.id=bd30687b-c884-4f01-8164-1b40abbc6bee version=v2.4.1 + + [vagrant@localhost ~]$ sudo netstat -tpnl | grep docker-proxy + tcp 0 0 10.64.33.81:5000 0.0.0.0:* LISTEN 27818/docker-proxy + + [vagrant@localhost ~]$ docker login -u admin -p admin123 10.64.33.81:5000 + Email: + WARNING: login credentials saved in /home/vagrant/.docker/config.json + Login Succeeded + + [vagrant@localhost ~]$ docker tag tangfeixiong/netcat-hello-http 10.64.33.81:5000/netcat-hello-http + + [vagrant@localhost ~]$ docker push 10.64.33.81:5000/netcat-hello-http + The push refers to a repository [10.64.33.81:5000/netcat-hello-http] + 5f70bf18a086: Pushed + 14996164fa4d: Pushed + 000ac623049f: Pushed + 3f50a8137adf: Pushed + latest: digest: sha256:e21c11edf699f9c582b46388f269a44ee5f0662b099cca7cfd30d5c6970cf563 size: 1748 + + [vagrant@localhost ~]$ ls /var/lib/registry/docker/registry/v2/repositories/ + netcat-hello-http + +### Install as `systemd` service + +In Official mirror repo + + [vagrant@localhost ~]$ sudo yum list docker-distribution + Loaded plugins: fastestmirror + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * extras: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + Available Packages + docker-distribution.x86_64 2.4.1-2.el7 extras + +Install from Official mirror + + [vagrant@localhost ~]$ sudo yum install docker-distribution + Loaded plugins: fastestmirror + base + centos-openshift-origin + extras + updates + Loading mirror speeds from cached hostfile + * base: mirrors.zju.edu.cn + * extras: mirrors.zju.edu.cn + * updates: mirrors.zju.edu.cn + Resolving Dependencies + --> Running transaction check + ---> Package docker-distribution.x86_64 0:2.4.1-2.el7 will be installed + --> Finished Dependency Resolution + + Dependencies Resolved + + ================================================================================ + Package Arch + ================================================================================ + Installing: + docker-distribution x86_6 + + Transaction Summary + ================================================================================ + Install 1 Package + +Service + + [vagrant@localhost ~]$ sudo vi /usr/lib/systemd/system/docker-distribution.service + [Unit] + Description=v2 Registry server for Docker + + [Service] + Type=simple + ExecStart=/usr/bin/registry serve /etc/docker-distribution/registry/config.yml + Restart=on-failure + + [Install] + WantedBy=multi-user.starget + +Configuration + + [vagrant@localhost ~]$ sudo vi /etc/docker-distribution/registry/config.yml + version: 0.1 + log: + fields: + service: registry + storage: + cache: + layerinfo: inmemory + filesystem: + rootdirectory: /var/lib/registry + http: + addr: :5000 + + [vagrant@localhost ~]$ sudo ls -d /var/lib/registry/ + /var/lib/registry/ diff --git a/hack/kubernetes-clearwater/scripts/centos-rsync.filter b/hack/kubernetes-clearwater/scripts/centos-rsync.filter new file mode 100755 index 0000000..5520cda --- /dev/null +++ b/hack/kubernetes-clearwater/scripts/centos-rsync.filter @@ -0,0 +1 @@ +# The CentOS-scoped filter \ No newline at end of file diff --git a/hack/kubernetes-clearwater/scripts/extras.exclude b/hack/kubernetes-clearwater/scripts/extras.exclude new file mode 100755 index 0000000..7c4dc30 --- /dev/null +++ b/hack/kubernetes-clearwater/scripts/extras.exclude @@ -0,0 +1,15 @@ +x86_64/drpms/** + +docker-unit-test-*.el7.centos.x86_64.rpm + +golang-pkg-darwin-*.el7.centos.x86_64.rpm +golang-pkg-freebsd-*.el7.centos.x86_64.rpm +golang-pkg-linux-386-*.el7.centos.x86_64.rpm +golang-pkg-linux-arm-*.el7.centos.x86_64.rpm +golang-pkg-netbsd-*.el7.centos.x86_64.rpm +golang-pkg-openbsd-*.el7.centos.x86_64.rpm +golang-pkg-plan9-*.el7.centos.x86_64.rpm +golang-pkg-windows-*.el7.centos.x86_64.rpm + +kubernetes-*.alpha1.*.el7.x86_64.rpm +kubernetes-unit-test-*.el7.x86_64.rpm \ No newline at end of file diff --git a/hack/kubernetes-clearwater/scripts/extras.include b/hack/kubernetes-clearwater/scripts/extras.include new file mode 100755 index 0000000..a2c4e9c --- /dev/null +++ b/hack/kubernetes-clearwater/scripts/extras.include @@ -0,0 +1 @@ +# The Extras-scoped includes \ No newline at end of file diff --git a/hack/kubernetes-clearwater/scripts/mirror-by-rsync.sh b/hack/kubernetes-clearwater/scripts/mirror-by-rsync.sh new file mode 100755 index 0000000..43ad278 --- /dev/null +++ b/hack/kubernetes-clearwater/scripts/mirror-by-rsync.sh @@ -0,0 +1,35 @@ +#!/bin/bash -e + +ALIYUN_MIRROR_URL="http://mirrors.aliyun.com/centos/7" +CENTOS_MIRROR_URL="http://mirror.centos.org/centos/7" +YUN_IDC_URL="rsync://mirrors.yun-idc.com/centos/7" + +SRC_URL="rsync://mirrors.yun-idc.com/centos/7" + +TGT=${SRC_URL//\//\%2F} +TGT=${TGT/\:/\%3A} + +DEST_HOME=$(dirname "${BASH_SOURCE}") + +if [[ ! -d $DEST_HOME/$TGT ]]; then mkdir -p $DEST_HOME/$TGT; fi + +#rsync -av \ +# --list-only \ +# --delete-excluded \ +# ${SRC_URL} + +# CentOS Extras +rsync -av --delete \ + --filter=". ${DEST_HOME}/centos-rsync.filter" \ + --filter=":n- .rsync.filter" \ + --exclude-from="${DEST_HOME}/extras.exclude" \ + --include-from="${DEST_HOME}/extras.include" \ + ${SRC_URL}/extras ${DEST_HOME}/$TGT + +# CentOS PaaS +rsync -av --delete \ + --filter=". ${DEST_HOME}/centos-rsync.filter" \ + --filter=":n- .rsync.filter" \ + --exclude-from="${DEST_HOME}/paas.exclude" \ + --include-from="${DEST_HOME}/paas.include" \ + ${SRC_URL}/paas ${DEST_HOME}/$TGT diff --git a/hack/kubernetes-clearwater/scripts/paas.exclude b/hack/kubernetes-clearwater/scripts/paas.exclude new file mode 100755 index 0000000..efc6164 --- /dev/null +++ b/hack/kubernetes-clearwater/scripts/paas.exclude @@ -0,0 +1,3 @@ +*-1.1.*.el7.x86_64.rpm +*-1.2.*.el7.x86_64.rpm +origin-tests-*.el7.x86_64.rpm \ No newline at end of file diff --git a/hack/kubernetes-clearwater/scripts/paas.include b/hack/kubernetes-clearwater/scripts/paas.include new file mode 100755 index 0000000..da5a9a6 --- /dev/null +++ b/hack/kubernetes-clearwater/scripts/paas.include @@ -0,0 +1 @@ +# The PaaS-scoped inlcudes \ No newline at end of file diff --git a/hack/kubernetes-clearwater/scripts/salt-make-ca-cert.sh b/hack/kubernetes-clearwater/scripts/salt-make-ca-cert.sh new file mode 100755 index 0000000..cf7c4bd --- /dev/null +++ b/hack/kubernetes-clearwater/scripts/salt-make-ca-cert.sh @@ -0,0 +1,26 @@ +#!/bin/bash -e + +#sudo groupadd -f -r kube-cert +CERT_DIR=$HOME +CERT_GRP=$(id -gn) + +MASTER_IP="10.64.33.81" +# Same as GKE, cluster CIDR: 10.120.0.0/14, service CIDR: 10.123.240.0/20 +MASTER_SERVICE="10.123.240.1" + +EXTRA_SANS=( + IP:$MASTER_IP + IP:$MASTER_SERVICE + DNS:kubernetes + DNS:kubernetes.default + DNS:kubernetes.default.svc + DNS:kubernetes.default.svc.cluster.local + ) + +ARG_CERT_IP=$MASTER_IP +ARG_EXTRA_SANS=$(echo "${EXTRA_SANS[@]}" | tr ' ' ,) + +echo $ARG_EXTRA_SANS +echo "Generate ca certs into $CERT_DIR" + +debug='true' CERT_DIR=$CERT_DIR CERT_GROUP=$CERT_GRP $HOME/kubernetes/saltbase/salt/generate-cert/make-ca-cert.sh $ARG_CERT_IP $ARG_EXTRA_SANS diff --git a/hack/sip-apps/blink-contact-editor.png b/hack/sip-apps/blink-contact-editor.png new file mode 100755 index 0000000..ff39403 Binary files /dev/null and b/hack/sip-apps/blink-contact-editor.png differ diff --git a/hack/sip-apps/blink-preferences-accounts-info.png b/hack/sip-apps/blink-preferences-accounts-info.png new file mode 100755 index 0000000..66c6f9b Binary files /dev/null and b/hack/sip-apps/blink-preferences-accounts-info.png differ diff --git a/hack/sip-apps/blink-preferences-accounts-media.png b/hack/sip-apps/blink-preferences-accounts-media.png new file mode 100755 index 0000000..d4b53fe Binary files /dev/null and b/hack/sip-apps/blink-preferences-accounts-media.png differ diff --git a/hack/sip-apps/blink-preferences-accounts-server.png b/hack/sip-apps/blink-preferences-accounts-server.png new file mode 100755 index 0000000..100bfad Binary files /dev/null and b/hack/sip-apps/blink-preferences-accounts-server.png differ diff --git a/hack/sip-apps/blink-preferences-advanced.png b/hack/sip-apps/blink-preferences-advanced.png new file mode 100755 index 0000000..7427a62 Binary files /dev/null and b/hack/sip-apps/blink-preferences-advanced.png differ diff --git a/hack/sip-apps/blink-preferences-audio.png b/hack/sip-apps/blink-preferences-audio.png new file mode 100755 index 0000000..242a181 Binary files /dev/null and b/hack/sip-apps/blink-preferences-audio.png differ diff --git a/hack/sip-apps/blink.png b/hack/sip-apps/blink.png new file mode 100755 index 0000000..198940d Binary files /dev/null and b/hack/sip-apps/blink.png differ diff --git a/hack/sip-apps/x-lite.png b/hack/sip-apps/x-lite.png new file mode 100755 index 0000000..1962a3c Binary files /dev/null and b/hack/sip-apps/x-lite.png differ diff --git a/hack/sip-apps/x-lite_account-settings_account.png b/hack/sip-apps/x-lite_account-settings_account.png new file mode 100755 index 0000000..71f84e2 Binary files /dev/null and b/hack/sip-apps/x-lite_account-settings_account.png differ diff --git a/hack/sip-apps/x-lite_account-settings_advanced.png b/hack/sip-apps/x-lite_account-settings_advanced.png new file mode 100755 index 0000000..7aa33c6 Binary files /dev/null and b/hack/sip-apps/x-lite_account-settings_advanced.png differ diff --git a/hack/sip-apps/x-lite_account-settings_topology.png b/hack/sip-apps/x-lite_account-settings_topology.png new file mode 100755 index 0000000..a1a3b24 Binary files /dev/null and b/hack/sip-apps/x-lite_account-settings_topology.png differ diff --git a/hack/sip-apps/x-lite_account-settings_transport.png b/hack/sip-apps/x-lite_account-settings_transport.png new file mode 100755 index 0000000..8029e01 Binary files /dev/null and b/hack/sip-apps/x-lite_account-settings_transport.png differ diff --git a/hack/sip-apps/x-lite_contact-profile.png b/hack/sip-apps/x-lite_contact-profile.png new file mode 100755 index 0000000..6c51fd0 Binary files /dev/null and b/hack/sip-apps/x-lite_contact-profile.png differ diff --git a/hack/sip-apps/zoiper-contact-editor.png b/hack/sip-apps/zoiper-contact-editor.png new file mode 100755 index 0000000..dfc19b0 Binary files /dev/null and b/hack/sip-apps/zoiper-contact-editor.png differ diff --git a/hack/sip-apps/zoiper-preferences-account-advanced.png b/hack/sip-apps/zoiper-preferences-account-advanced.png new file mode 100755 index 0000000..1d63925 Binary files /dev/null and b/hack/sip-apps/zoiper-preferences-account-advanced.png differ diff --git a/hack/sip-apps/zoiper-preferences-account-codec.png b/hack/sip-apps/zoiper-preferences-account-codec.png new file mode 100755 index 0000000..33891cf Binary files /dev/null and b/hack/sip-apps/zoiper-preferences-account-codec.png differ diff --git a/hack/sip-apps/zoiper-preferences-account-extra.png b/hack/sip-apps/zoiper-preferences-account-extra.png new file mode 100755 index 0000000..f1ddb3a Binary files /dev/null and b/hack/sip-apps/zoiper-preferences-account-extra.png differ diff --git a/hack/sip-apps/zoiper-preferences-account-general.png b/hack/sip-apps/zoiper-preferences-account-general.png new file mode 100755 index 0000000..99ab756 Binary files /dev/null and b/hack/sip-apps/zoiper-preferences-account-general.png differ diff --git a/hack/sip-apps/zoiper-preferences-advanced-network.png b/hack/sip-apps/zoiper-preferences-advanced-network.png new file mode 100755 index 0000000..1e1f163 Binary files /dev/null and b/hack/sip-apps/zoiper-preferences-advanced-network.png differ diff --git a/hack/sip-apps/zoiper.png b/hack/sip-apps/zoiper.png new file mode 100755 index 0000000..24fee2a Binary files /dev/null and b/hack/sip-apps/zoiper.png differ