k8s基于calico NetworkPolicy访问限制测试

### master 所需软件包
[root@k8s-master ~]# rpm -qa | grep kube
kubernetes-client-1.8.4-1.el7.centos.x86_64
kubernetes-master-1.8.4-1.el7.centos.x86_64
etcd-3.2.7-1.el7.x86_64
### 相关配置文件如下

[root@k8s-master ~]# grep -Ev '^$|^#' /etc/kubernetes/apiserver
KUBE_API_ADDRESS="--address=0.0.0.0"
KUBE_API_PORT="--port=8080"
KUBE_ETCD_SERVERS="--etcd_servers=http://10.30.56.108:2379"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"
KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ResourceQuota"
KUBE_API_ARGS=" --cors_allowed_origins=.*  --service-node-port-range='8000-32767' --runtime-config=extensions/v1beta1 "

[root@k8s-master ~]# grep -Ev '^$|^#' /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=true"
KUBE_MASTER="--master=http://127.0.0.1:8080"
[root@k8s-master ~]# grep -Ev '^$|^#' /etc/kubernetes/controller-manager
KUBE_CONTROLLER_MANAGER_ARGS="--address=0.0.0.0 \
                              --service-cluster-ip-range=10.254.0.0/16 \
                              --cluster-name=kubernetes \
                              --leader-elect=true \
                              --node-monitor-grace-period=40s \
                              --node-monitor-period=5s \
                              --pod-eviction-timeout=50ms"
[root@k8s-master ~]#
KUBE_SCHEDULER_ARGS="--leader-elect=true"

[root@k8s-master ~]# grep -Ev '^$|^#' /etc/kubernetes/proxy
KUBE_PROXY_ARGS="--proxy-mode=iptables"

[root@k8s-master ~]# grep -Ev '^$|^#' /etc/etcd/etcd.conf
ETCD_NAME=default
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_CLIENT_URLS="http://localhost:2379,http://10.30.56.108:2379"
ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379,http://10.30.56.108:2379"

### node所需k8s docker软件包
[root@k8s-node001 ~]# rpm -qa | grep docker
docker-1.12.6-61.git85d7426.el7.centos.x86_64
docker-client-1.12.6-61.git85d7426.el7.centos.x86_64
docker-common-1.12.6-61.git85d7426.el7.centos.x86_64
[root@k8s-node001 ~]# rpm -qa | grep kube
kubernetes-node-1.8.4-1.el7.centos.x86_64
kubernetes-client-1.8.4-1.el7.centos.x86_64


### node上kubelet相关配置
[root@k8s-node001 ~]# grep -Ev '^$|^#' /etc/kubernetes
kubernetes/     kubernetes.bak/
[root@k8s-node001 ~]# grep -Ev '^$|^#' /etc/kubernetes/
config              controller-manager  kubelet             kubelet.kubeconfig  proxy
[root@k8s-node001 ~]# grep -Ev '^$|^#' /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=0"
KUBE_ALLOW_PRIV="--allow-privileged=true"
KUBE_MASTER="--master=http://10.30.56.108:8080"
[root@k8s-node001 ~]# grep -Ev '^$|^#' /etc/kubernetes/controller-manager
KUBE_CONTROLLER_MANAGER_ARGS="--address=0.0.0.0 \
                              --service-cluster-ip-range=10.254.0.0/16 \
                              --cluster-name=kubernetes \
                              --leader-elect=true \
                              --node-monitor-grace-period=40s \
                              --node-monitor-period=5s \
                              --pod-eviction-timeout=50ms"
[root@k8s-node001 ~]# grep -Ev '^$|^#' /etc/kubernetes/kubelet
KUBELET_ADDRESS="--address=0.0.0.0"
KUBELET_PORT="--port=10250"
KUBELET_HOSTNAME=""
KUBELET_API_SERVER="--kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
KUBELET_ARGS=" --cgroup-driver=systemd --docker-root=/home/docker --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d --pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0"
[root@k8s-node001 ~]# grep -Ev '^$|^#' /etc/kubernetes/kubelet.kubeconfig
apiVersion: v1
kind: Config
clusters:
  - cluster:
      server: http://10.30.56.108:8080/
    name: local
contexts:
  - context:
      cluster: local
    name: local
current-context: local
[root@k8s-node001 ~]# grep -Ev '^$|^#' /etc/kubernetes/proxy
KUBE_PROXY_ARGS="--proxy-mode=iptables"


### 注意docker段地址
[root@k8s-node001 kubernetes]# cat  /etc/sysconfig/docker
# /etc/sysconfig/docker
# #
# # Other arguments to pass to the docker daemon process
# # These will be parsed by the sysv initscript and appended
# # t the arguments list passed to docker -d
OPTIONS=" --exec-opt native.cgroupdriver=cgroupfs --graph=/home/docker --default-ulimit nproc=65535:65535 --default-ulimit nofile=655350:655350 --storage-opt dm.basesize=40G --storage-opt dm.fs=ext4 --storage-driver=devicemapper --storage-opt dm.mountopt=nodiscard --storage-opt dm.blkdiscard=false --bip=172.27.100.1/24"
INSECURE_REGISTRY=" --insecure-registry docker"

### 同时  node节点需要安装 bird软件路由器配合交换机层做bgp协议
[root@k8s-node001 kubernetes]# rpm -qa | grep bird
bird-1.4.5-2.el7.x86_64
bird6-1.4.5-2.el7.x86_64
[root@k8s-node001 kubernetes]# cat /etc/bird.conf
log syslog all;

### 下面这个id 为 node节点 物理网卡IP
router id 10.88.200.31;  

# Define a route filter...
filter test_filter {
    if net = 172.27.100.0/24 then accept;  ### 注意 这个 网段需要和docker0的网段以及下面的calico定义的node所使用的网段在一个段
else reject;
}

# Define another routing table
#table main;

protocol direct {
    interface "*"; # Restrict network interfaces it works with
}

protocol kernel {
    import all;
    learn; # Learn all alien routes from the kernel
  # persist; # Don't remove routes on bird shutdown
    scan time 20; # Scan kernel routing table every 20 seconds
  # import none; # Default is import all
    export all; # Default is export none
  # kernel table 254; # Kernel table to synchronize with (default: ain)
}

protocol device {
    scan time 10; # Scan interfaces every 10 seconds
}

protocol static {
   # import all;
   # export all;
   # table main; # Connect to a non-default table
     route 1.1.1.1:255.255.255.255 via "eth0";
}

# Pipe protocol connects two routing tables... Beware of loops.
#protocol pipe {
   # peer table testable;
   # Define what routes do we export to this protocol / import from it.
   # import all; # default is all
   # export all; # default is none
   # import none; # If you wish to disable imports
   # import filter test_filter; # Use named filter
   # import where source = RTS_DEVICE; # Use explicit filter
#}

protocol ospf MyOSPF {
   # tick 2;
     rfc1583compat yes;
     import filter test_filter;
     export filter test_filter;
     area 0.0.0.0 {
        stub no;
        interface "bond0" {
           hello 2;
           retransmit 6;
           cost 10;
           transmit delay 5;
           dead count 4;
           wait 50;
           type broadcast;
           authentication simple;
           password "momo1602";
       };
       interface "docker0" {
           hello 2;
           retransmit 6;
           cost 10;
           transmit delay 5;
           dead count 4;
           wait 50;
           type broadcast;
           authentication simple;
           password "momo1602";
       };
    };
}

###  node节点 calico所需相关配置以及二进制文件
####  calicoctl命令行工具
wget https://www.projectcalico.org/builds/calicoctl
chmod +x calicoctl
ln -s ./calicoctl /usr/bin/calicoctl
#### calico以及以及calico-ipam
wget -N -P /opt/cni/bin https://github.com/projectcalico/cni-plugin/releases/download/v1.11.0/calico
wget -N -P /opt/cni/bin https://github.com/projectcalico/cni-plugin/releases/download/v1.11.0/calico-ipam
chmod +x /opt/cni/bin/calico /opt/cni/bin/calico-ipam
#### loopback 
wget https://github.com/containernetworking/cni/releases/download/v0.3.0/cni-v0.3.0.tgz
tar -zxvf cni-v0.3.0.tgz
cp loopback /opt/cni/bin/
### calico相关文件下载完后，放在指定目录下 
[root@k8s-node001 net.d]# pwd
/etc/cni/net.d
[root@k8s-node001 net.d]# cat 10-calico.conf
{
    "name": "k8s-pod-network",
    "cniVersion": "0.3.0",
    "type": "calico",
    "etcd_endpoints": "http://10.30.56.108:2379",
    "mtu": 1500,
    "ipam": {
        "type": "host-local",
        "subnet": "172.27.100.0/27"     ###  注意 这个位置的网段，即为 该node能够给容器分配的网段，需要和docker0 以及 bird配置中的net accept的网段保持一致。
    },
    "policy": {
        "type": "k8s",
        "k8s_api_root": "http://10.30.56.108:8080"
    },
    "kubernetes": {
        "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
    }
}
[root@k8s-node001 net.d]# cat calico-kubeconfig
# Kubeconfig file for Calico CNI plugin.
apiVersion: v1
kind: Config
clusters:
- name: local
  cluster:
    insecure-skip-tls-verify: true
users:
- name: calico
contexts:
- name: calico-context
  context:
    cluster: local
    user: calico
current-context: calico-context
[root@k8s-node001 bin]# pwd
/opt/cni/bin
[root@k8s-node001 bin]# ll
total 68168
-rwxr-xr-x 1 root root 29062464 Dec  7 19:09 calico
-rwxr-xr-x 1 root root 28424000 Dec  7 19:09 calico-ipam
-rwxr-xr-x 1 root root  2814104 Dec  7 19:09 flannel
-rwxr-xr-x 1 root root  2991965 Dec  7 19:09 host-local
-rwxr-xr-x 1 root root  3026388 Dec  7 19:09 loopback
-rwxr-xr-x 1 root root  3470464 Dec  7 19:09 portmap

###  node节点使用systemd方式部署calico-node
[root@k8s-node001 ~]# cat /etc/systemd/system/calico-node.service
[Unit]
Description=calico node
After=docker.service
Requires=docker.service

[Service]
User=root
Environment=ETCD_ENDPOINTS=http://10.30.56.108:2379
PermissionsStartOnly=true
ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \
  -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
  -e IP= \
  -e AS= \
  -e CALICO_LIBNETWORK_ENABLED=true \
  -e NO_DEFAULT_POOLS=true \
  -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \
  -e CALICO_LIBNETWORK_CREATE_PROFILES=false \
  -e IP6= \
  -v /var/run/calico:/var/run/calico \
  -v /lib/modules:/lib/modules \
  -v /run/docker/plugins:/run/docker/plugins \
  -v /var/run/docker.sock:/var/run/docker.sock \
  -v /var/log/calico:/var/log/calico \
  calico/node:v1.0.2
ExecStop=/usr/bin/docker rm -f calico-node
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target

for i in {kube-controller-manager,kube-scheduler,kube-apiserver,etcd};do systemctl start ${i} && systemctl enable ${i} && systemctl status ${i}; done
[root@k8s-master ~]# netstat -tunlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 127.0.0.1:2380          0.0.0.0:*               LISTEN      19222/etcd
tcp        0      0 10.30.56.108:22         0.0.0.0:*               LISTEN      11748/sshd
tcp        0      0 10.30.56.108:10050      0.0.0.0:*               LISTEN      22174/zabbix_agentd
tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      19222/etcd
tcp        0      0 10.30.56.108:2379       0.0.0.0:*               LISTEN      19222/etcd
tcp6       0      0 :::10252                :::*                    LISTEN      27251/kube-controll
tcp6       0      0 :::8080                 :::*                    LISTEN      27714/kube-apiserve
tcp6       0      0 :::6443                 :::*                    LISTEN      27714/kube-apiserve
tcp6       0      0 :::10251                :::*                    LISTEN      27252/kube-schedule
udp        0      0 0.0.0.0:68              0.0.0.0:*                           715/dhclient
udp        0      0 10.30.56.108:123        0.0.0.0:*                           10048/ntpd
udp        0      0 127.0.0.1:123           0.0.0.0:*                           10048/ntpd
udp        0      0 127.0.0.1:514           0.0.0.0:*                           2716/rsyslogd
udp        0      0 0.0.0.0:22237           0.0.0.0:*                           715/dhclient
udp6       0      0 :::64756                :::*                                715/dhclient

 for i in {bird,kubelet,calico-node};do systemctl start ${i} && systemctl enable ${i} && systemctl status ${i}; done
 [root@k8s-node001 ~]# netstat -tunlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 127.0.0.1:10248         0.0.0.0:*               LISTEN      97080/kubelet
tcp        0      0 10.88.200.31:22         0.0.0.0:*               LISTEN      56104/sshd
tcp        0      0 10.88.200.31:10050      0.0.0.0:*               LISTEN      87871/zabbix_agentd
tcp6       0      0 :::10250                :::*                    LISTEN      97080/kubelet
tcp6       0      0 :::10255                :::*                    LISTEN      97080/kubelet
tcp6       0      0 :::4194                 :::*                    LISTEN      97080/kubelet
udp        0      0 10.88.200.31:123        0.0.0.0:*                           67091/ntpd
udp        0      0 127.0.0.1:123           0.0.0.0:*                           67091/ntpd
udp        0      0 127.0.0.1:514           0.0.0.0:*                           2090/rsyslogd

[root@k8s-master ~]# kubectl get nodes
NAME                        STATUS    ROLES     AGE       VERSION
k8s-node001   Ready     <none>    9d        v1.8.4
k8s-node002   Ready     <none>    9d        v1.8.4
[root@k8s-node001 ~]# calicoctl node status
Calico process is running.

IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+--------------+-------------------+-------+----------+-------------+
| 10.88.200.32 | node-to-node mesh | up    | 04:33:22 | Established |
+--------------+-------------------+-------+----------+-------------+
[root@k8s-node002 ~]# calicoctl node status
Calico process is running.

IPv4 BGP status
+--------------+-------------------+-------+----------+-------------+
| PEER ADDRESS |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+--------------+-------------------+-------+----------+-------------+
| 10.88.200.31 | node-to-node mesh | up    | 04:33:22 | Established |
+--------------+-------------------+-------+----------+-------------+