Kubernetes

[K8S Deploy 6주차] 폐쇄망에서 kubespray를 이용한 K8S 구성

yu3papa 2026. 2. 14. 10:13

이번 블로그에서는 인터넷이 제한되는 폐쇄망 환경에서 kubespray를 이용하여 쿠버네티스 클러스트를 구성하는 실습을 진행합니다.

 

실습 환경 구성

Vargant를 이용하여 Rocky Linux 10 운영체제가 설치된 3대의 VM을 생성합니다.

  • Vagrant 관련 파일
더보기
  • Vagrantfile
# Base Image  https://portal.cloud.hashicorp.com/vagrant/discover/bento/rockylinux-10.0
BOX_IMAGE = "bento/rockylinux-10.0" # "bento/rockylinux-9"
BOX_VERSION = "202510.26.0"
N = 2 # max number of Node

Vagrant.configure("2") do |config|

# Nodes 
  (1..N).each do |i|
    config.vm.define "k8s-node#{i}" do |subconfig|
      subconfig.vm.box = BOX_IMAGE
      subconfig.vm.box_version = BOX_VERSION
      subconfig.vm.provider "virtualbox" do |vb|
        vb.customize ["modifyvm", :id, "--groups", "/Kubespary-offline-Lab"]
        vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
        vb.name = "k8s-node#{i}"
        vb.cpus = 4
        vb.memory = 2048
        vb.linked_clone = true
      end
      subconfig.vm.host_name = "k8s-node#{i}"
      subconfig.vm.network "private_network", ip: "192.168.10.1#{i}"
      subconfig.vm.network "forwarded_port", guest: 22, host: "6000#{i}", auto_correct: true, id: "ssh"
      subconfig.vm.synced_folder "./", "/vagrant", disabled: true
      subconfig.vm.provision "shell", path: "init_cfg.sh" , args: [ N ]
    end
  end

# Admin Node
    config.vm.define "admin" do |subconfig|
      subconfig.vm.box = BOX_IMAGE
      subconfig.vm.box_version = BOX_VERSION
      subconfig.vm.provider "virtualbox" do |vb|
        vb.customize ["modifyvm", :id, "--groups", "/Kubespary-offline-Lab"]
        vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
        vb.name = "admin"
        vb.cpus = 4
        vb.memory = 2048
        vb.linked_clone = true
      end
      subconfig.vm.host_name = "admin"
      subconfig.vm.network "private_network", ip: "192.168.10.10"
      subconfig.vm.network "forwarded_port", guest: 22, host: "60000", auto_correct: true, id: "ssh"
      subconfig.vm.synced_folder "./", "/vagrant", disabled: true
      subconfig.vm.disk :disk, size: "120GB", primary: true # https://developer.hashicorp.com/vagrant/docs/disks/usage
      subconfig.vm.provision "shell", path: "admin.sh" , args: [ N ]
    end

end
  • admin.sh
#!/usr/bin/env bash

echo ">>>> Initial Config Start <<<<"


echo "[TASK 1] Change Timezone and Enable NTP"
timedatectl set-local-rtc 0
timedatectl set-timezone Asia/Seoul


echo "[TASK 2] Disable firewalld and selinux"
systemctl disable --now firewalld >/dev/null 2>&1
setenforce 0
sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config


echo "[TASK 3] Setting Local DNS Using Hosts file"
sed -i '/^127\.0\.\(1\|2\)\.1/d' /etc/hosts
echo "192.168.10.10 admin" >> /etc/hosts
for (( i=1; i<=$1; i++  )); do echo "192.168.10.1$i k8s-node$i" >> /etc/hosts; done


echo "[TASK 4] Delete default routing - enp0s9 NIC" # setenforce 0 설정 필요
nmcli connection modify enp0s9 ipv4.never-default yes
nmcli connection up enp0s9 >/dev/null 2>&1


echo "[TASK 5] Config net.ipv4.ip_forward"
cat << EOF > /etc/sysctl.d/99-ipforward.conf
net.ipv4.ip_forward = 1
EOF
sysctl --system  >/dev/null 2>&1


echo "[TASK 6] Install packages"
dnf install -y python3-pip git sshpass cloud-utils-growpart >/dev/null 2>&1


echo "[TASK 7] Install Helm"
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | DESIRED_VERSION=v3.20.0 bash >/dev/null 2>&1


echo "[TASK 8] Increase Disk Size"
growpart /dev/sda 3 >/dev/null 2>&1 # lsblk
xfs_growfs /dev/sda3 >/dev/null 2>&1 # df -hT /


echo "[TASK 9] Setting SSHD"
echo "root:qwe123" | chpasswd

cat << EOF >> /etc/ssh/sshd_config
PermitRootLogin yes
PasswordAuthentication yes
EOF
systemctl restart sshd >/dev/null 2>&1


echo "[TASK 10] Setting SSH Key"
ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa >/dev/null 2>&1
sshpass -p 'qwe123' ssh-copy-id -o StrictHostKeyChecking=no root@192.168.10.10  >/dev/null 2>&1  # cat /root/.ssh/authorized_keys
ssh -o StrictHostKeyChecking=no root@admin-lb hostname >/dev/null 2>&1
for (( i=1; i<=$1; i++  )); do sshpass -p 'qwe123' ssh-copy-id -o StrictHostKeyChecking=no root@192.168.10.1$i >/dev/null 2>&1 ; done
for (( i=1; i<=$1; i++  )); do sshpass -p 'qwe123' ssh -o StrictHostKeyChecking=no root@k8s-node$i hostname >/dev/null 2>&1 ; done


echo "[TASK 11] Install K9s"
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
wget -P /tmp https://github.com/derailed/k9s/releases/latest/download/k9s_linux_${CLI_ARCH}.tar.gz  >/dev/null 2>&1
tar -xzf /tmp/k9s_linux_${CLI_ARCH}.tar.gz -C /tmp
chown root:root /tmp/k9s
mv /tmp/k9s /usr/local/bin/
chmod +x /usr/local/bin/k9s


echo "[TASK 12] ETC"
echo "sudo su -" >> /home/vagrant/.bashrc


echo ">>>> Initial Config End <<<<"
  • init_cfg.sh
#!/usr/bin/env bash

echo ">>>> Initial Config Start <<<<"


echo "[TASK 1] Change Timezone and Enable NTP"
timedatectl set-local-rtc 0
timedatectl set-timezone Asia/Seoul


echo "[TASK 2] Disable firewalld and selinux"
systemctl disable --now firewalld >/dev/null 2>&1
setenforce 0
sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config


echo "[TASK 3] Disable and turn off SWAP & Delete swap partitions"
swapoff -a
sed -i '/swap/d' /etc/fstab
sfdisk --delete /dev/sda 2 >/dev/null 2>&1
partprobe /dev/sda >/dev/null 2>&1


echo "[TASK 4] Config kernel & module"
cat << EOF > /etc/modules-load.d/k8s.conf
overlay
br_netfilter
vxlan
EOF
modprobe overlay >/dev/null 2>&1
modprobe br_netfilter >/dev/null 2>&1

cat << EOF >/etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sysctl --system >/dev/null 2>&1


echo "[TASK 5] Setting Local DNS Using Hosts file"
sed -i '/^127\.0\.\(1\|2\)\.1/d' /etc/hosts
echo "192.168.10.10 admin" >> /etc/hosts
for (( i=1; i<=$1; i++  )); do echo "192.168.10.1$i k8s-node$i" >> /etc/hosts; done


echo "[TASK 6] Delete default routing - enp0s9 NIC" # setenforce 0 설정 필요
nmcli connection modify enp0s9 ipv4.never-default yes
nmcli connection up enp0s9 >/dev/null 2>&1


echo "[TASK 7] Setting SSHD"
echo "root:qwe123" | chpasswd

cat << EOF >> /etc/ssh/sshd_config
PermitRootLogin yes
PasswordAuthentication yes
EOF
systemctl restart sshd >/dev/null 2>&1


echo "[TASK 8] Install packages"
dnf install -y python3-pip git >/dev/null 2>&1


echo "[TASK 9] ETC"
echo "sudo su -" >> /home/vagrant/.bashrc


echo ">>>> Initial Config End <<<<"

위 3개의 파일을 이용하여 VM 생성을 진행합니다.

mkdir k8s-offline
cd k8s-offline

curl -O https://raw.githubusercontent.com/gasida/vagrant-lab/refs/heads/main/k8s-kubespary-offline/Vagrantfile
curl -O https://raw.githubusercontent.com/gasida/vagrant-lab/refs/heads/main/k8s-kubespary-offline/admin.sh
curl -O https://raw.githubusercontent.com/gasida/vagrant-lab/refs/heads/main/k8s-kubespary-offline/init_cfg.sh

vagrant up

# 생성된 VM 확인
vagrant status
k8s-node1                 running (virtualbox)
k8s-node2                 running (virtualbox)
admin                     running (virtualbox)

# root 계정의 암호인 qwe123 을 이용하여 SSH 접속
ssh root@192.168.10.10
ssh root@192.168.10.11
ssh root@192.168.10.12

 

3대의 vm 을 생성하면 네트워크 구조는 아래와 같습니다.

모든 node에서 enp0s3를 NIC를 이용하여 인터넷을 할 수 있는 환경입니다.

[root@admin ~]# ip -br a
lo               UNKNOWN        127.0.0.1/8 ::1/128
enp0s3           UP             10.0.2.15/24 fd17:625c:f037:2:a00:27ff:fef8:377b/64 fe80::a00:27ff:fef8:377b/64
enp0s8           UP             192.168.10.10/24 fe80::61b5:58db:ba44:e59d/64

[root@k8s-node1 ~]# ip -br a
lo               UNKNOWN        127.0.0.1/8 ::1/128
enp0s3           UP             10.0.2.15/24 fd17:625c:f037:2:a00:27ff:fef8:377b/64 fe80::a00:27ff:fef8:377b/64
enp0s8           UP             192.168.10.11/24 fe80::4b92:a18d:af6d:2b1e/64

[root@k8s-node2 ~]# ip -br a
lo               UNKNOWN        127.0.0.1/8 ::1/128
enp0s3           UP             10.0.2.15/24 fd17:625c:f037:2:a00:27ff:fef8:377b/64 fe80::a00:27ff:
enp0s8           UP             192.168.10.12/24 fe80::70cf:7f65:c231:ef8a/64

 

NAT Gateway

NAT Gateway(Network Address Translation Gateway)는 프라이빗 서브넷(Private Subnet)에 있는 리소스가 인터넷에 연결될 수 있도록 돕는 동시에, 외부 인터넷에서 해당 리소스로 직접 접근하는 것은 차단하는 보안 및 연결 전용 게이트웨이입니다.

AWS 의 NAT Gateway 예시

k8s-node1, k8s-node2 머신에서 디폴트 라우팅을 변경하여 인터넷이 되지 않도록 설정합니다.

# enp0s3 연결 다운 : 실행 직후 부터 외부 인터넷으로 연결이 안됩니다.
nmcli connection down enp0s3
Connection 'enp0s3' successfully deactivated (D-Bus active path: /org/freedesktop/NetworkManager/ActiveConnection/2)

# enp0s3 확인 : 할당된 IP가 제거되고, 외부 통신 라우팅 정보도 삭제됨 
cat /etc/NetworkManager/system-connections/enp0s3.nmconnection
[connection]
id=enp0s3
uuid=74d64498-c95a-435b-9340-81b2e4a7d1f2
type=ethernet
interface-name=enp0s3

[ethernet]

[ipv4]
method=auto

[ipv6]
addr-gen-mode=eui64
method=ignore

[proxy]

ip addr show enp0s3
2: enp0s3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 08:00:27:f8:37:7b brd ff:ff:ff:ff:ff:ff
    altname enx080027f8377b

# 이제 외부 인터넷을 연결되지 않습니다.
ping -c 1 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.

--- 8.8.8.8 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms


# 재부팅 이후에도 enp0s3 NIC가 "자동 연결" 되지 않게 설정
nmcli connection modify enp0s3 connection.autoconnect no


# 외부 통신을 위해 enp0s8 에 디폴트 라우팅을 인터넷이 되는 admin 노드(192.168.10.10) 추가 : 우선순위 200 설정
nmcli connection modify enp0s8 +ipv4.routes "0.0.0.0/0 192.168.10.10 200"
nmcli connection up enp0s8
Connection successfully activated (D-Bus active path: /org/freedesktop/NetworkManager/ActiveConnection/6)

ip route
default via 192.168.10.1 dev enp0s8 proto static metric 100
default via 192.168.10.10 dev enp0s8 proto static metric 200
192.168.10.0/24 dev enp0s8 proto kernel scope link src 192.168.10.11 metric 100

# 192.168.10.1 으로 향하는 default 라우팅을 제거합니다.
ip route del default via 192.168.10.1

# Network Manager가 DNS정보를 가져오는데 default 를 꺼서 DNS 설정이 없어서 /etc/resolve.con 파일을 별도로 수정
cat << EOF > /etc/resolv.conf
nameserver 168.126.63.1
nameserver 8.8.8.8
EOF

# 현재는 외부 인터넷이 되지 않는 상태입니다.
curl www.google.com
curl: (6) Could not resolve host: http://www.google.com

 

이제 인터넷이 되는  admin 서버에서 NAT Gateway 설정을 진행합니다.

# NAT Gateway 역할을 수행하려면 net.ipv4.ip_forward 커널 파라미터가 1 로 설정되어야 합니다.
sysctl -w net.ipv4.ip_forward=1 

cat <<EOF | tee /etc/sysctl.d/99-ipforward.conf
net.ipv4.ip_forward = 1
EOF

sysctl --system

# NAT 설정 : 인터넷이 가능한 enp0s3 인터페이스에 MASQUERADE 규칙을 추가
iptables -t nat -A POSTROUTING -o enp0s3 -j MASQUERADE

# 이제 k8s-node1, 2 에서 admin 노드를 NAT Gateway 로 이용하여 인터넷에 연결이 가능합니다.
[root@k8s-node1 ~]# ping -c 2 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.
64 bytes from 8.8.8.8: icmp_seq=1 ttl=254 time=40.2 ms
64 bytes from 8.8.8.8: icmp_seq=2 ttl=254 time=41.0 ms

--- 8.8.8.8 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1298ms
rtt min/avg/max/mdev = 40.167/40.598/41.029/0.431 ms

# 폐쇄망 환경 구성을 위해 k8s-node1,2 에서는 인터넷이 되지 않도록 admin 노드에서 iptables 규칙 제
iptables -t nat -D POSTROUTING -o enp0s3 -j MASQUERADE

# 이제 k8s-node1,2 노드에서는 인터넷이 되지 않습니다.
[root@k8s-node1 ~]# ping -c 2 8.8.8.8
PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.

--- 8.8.8.8 ping statistics ---
2 packets transmitted, 0 received, 100% packet loss, time 1171ms

 

이제 서버 환경 구성은 아래와 같이 변경되었습니다.

NTP 서버 설정

admin 서버는 인터넷이 되는 환경이어서 ntp client 역할을 수행하면 시간동기화를 수행합니다.

더불어 인터넷이 안되는 k8s-node1, 2 를 위해 ntp 서버 역할도 수행해야 합니다.

  • admin 서버에서 수행
# chrony 설정
cp /etc/chrony.conf /etc/chrony.bak
cat << EOF > /etc/chrony.conf
# 외부 한국 공용 NTP 서버 설정
server pool.ntp.org iburst
server kr.pool.ntp.org iburst

# 내부망(192.168.10.0/24)에서 이 서버에 접속하여 시간 동기화 허용
allow 192.168.10.0/24

# 외부망이 끊겼을 때도 로컬 시계를 기준으로 내부망에 시간 제공 (선택 사항)
local stratum 10

# 로그
logdir /var/log/chrony
EOF

systemctl restart chronyd.service
systemctl status chronyd.service --no-pager

# 상태 확인
timedatectl status
               Local time: Sat 2026-02-14 11:42:07 KST
           Universal time: Sat 2026-02-14 02:42:07 UTC
                 RTC time: Sat 2026-02-14 02:38:02
                Time zone: Asia/Seoul (KST, +0900)
System clock synchronized: yes
              NTP service: active
          RTC in local TZ: no

chronyc sources -v
  .-- Source mode  '^' = server, '=' = peer, '#' = local clock.
 / .- Source state '*' = current best, '+' = combined, '-' = not combined,
| /             'x' = may be in error, '~' = too variable, '?' = unusable.
||                                                 .- xxxx [ yyyy ] +/- zzzz
||      Reachability register (octal) -.           |  xxxx = adjusted offset,
||      Log2(Polling interval) --.      |          |  yyyy = measured offset,
||                                \     |          |  zzzz = estimated error.
||                                 |    |           \
MS Name/IP address         Stratum Poll Reach LastRx Last sample
===============================================================================
^- 121.174.142.82                3   6    17    34  +1266us[+1266us] +/-   31ms
^* 121.134.215.104               2   6    17    34  -1202us[-1607us] +/- 5006us

 

  • k8s-node1,2 에서는 admin 서버를 시간동기화 서버로 설정합니다.
# chrony 설정
cp /etc/chrony.conf /etc/chrony.bak
cat << EOF > /etc/chrony.conf
server 192.168.10.10 iburst
logdir /var/log/chrony
EOF

systemctl restart chronyd.service

# 상태 확인
timedatectl status
               Local time: Sat 2026-02-14 11:44:55 KST
           Universal time: Sat 2026-02-14 02:44:55 UTC
                 RTC time: Sat 2026-02-14 02:43:44
                Time zone: Asia/Seoul (KST, +0900)
System clock synchronized: yes
              NTP service: active
          RTC in local TZ: no


chronyc sources -v
  .-- Source mode  '^' = server, '=' = peer, '#' = local clock.
 / .- Source state '*' = current best, '+' = combined, '-' = not combined,
| /             'x' = may be in error, '~' = too variable, '?' = unusable.
||                                                 .- xxxx [ yyyy ] +/- zzzz
||      Reachability register (octal) -.           |  xxxx = adjusted offset,
||      Log2(Polling interval) --.      |          |  yyyy = measured offset,
||                                \     |          |  zzzz = estimated error.
||                                 |    |           \
MS Name/IP address         Stratum Poll Reach LastRx Last sample
===============================================================================
^* admin                         3   6    17    47  +1084us[+2694us] +/- 8264us


# admin 서버에서 자신의 NTP Server 를 사용하는 클라이언트 확인
chronyc clients
Hostname                      NTP   Drop Int IntL Last     Cmd   Drop Int  Last
===============================================================================
k8s-node1                       5      0   4   -    31       0      0   -     -
k8s-node2                       8      0   4   -    23       0      0   -     -

 

DNS 서버 설정

인터넷이 되는 admin 서버가 dns 서버 역할을 하고, k8s-node1,2는 인터넷이 안되는 환경이어서 dns 서버를 admin 서버로 설정합니다.

  • admin 서버에 DNS 서버 역할 추가
# bind 설치
dnf install -y bind bind-utils

# named.conf 설정
cp /etc/named.conf /etc/named.bak
cat <<EOF > /etc/named.conf
options {
        listen-on port 53 { any; };
        listen-on-v6 port 53 { ::1; };
        directory       "/var/named";
        dump-file       "/var/named/data/cache_dump.db";
        statistics-file "/var/named/data/named_stats.txt";
        memstatistics-file "/var/named/data/named_mem_stats.txt";
        secroots-file   "/var/named/data/named.secroots";
        recursing-file  "/var/named/data/named.recursing";
        allow-query     { 127.0.0.1; 192.168.10.0/24; };
        allow-recursion { 127.0.0.1; 192.168.10.0/24; };

        forwarders {
                168.126.63.1;
                8.8.8.8;
        };

        recursion yes;

        dnssec-validation auto;  # https://sirzzang.github.io/kubernetes/Kubernetes-Kubespray-08-01-06/#troubleshooting-dnssec-%EA%B2%80%EC%A6%9D-%EC%8B%A4%ED%8C%A8

        managed-keys-directory "/var/named/dynamic";
        geoip-directory "/usr/share/GeoIP";

        pid-file "/run/named/named.pid";
        session-keyfile "/run/named/session.key";

        include "/etc/crypto-policies/back-ends/bind.config";
};

logging {
        channel default_debug {
                file "data/named.run";
                severity dynamic;
        };
};

zone "." IN {
        type hint;
        file "named.ca";
};

include "/etc/named.rfc1912.zones";
include "/etc/named.root.key";
EOF


# 문법 오류 확인 (아무 메시지 없으면 정상)
named-checkconf /etc/named.conf

# 서비스 활성화 및 시작
systemctl enable --now named

# DMZ 서버 자체 DNS 설정 (자기 자신 사용)
echo "nameserver 192.168.10.10" > /etc/resolv.conf

# 확인
dig +short google.com
172.217.211.139
172.217.211.138
172.217.211.100
172.217.211.102
172.217.211.113
172.217.211.101

# NetworkManager에서 DNS 관리 끄기 : 미적용 시, admin 서버 재부팅 시 NetworkManager 가 초기 설정값 덮어쓰움.
cat << EOF > /etc/NetworkManager/conf.d/99-dns-none.conf
[main]
dns=none
EOF
systemctl restart NetworkManager

 

  • k8s-node1,2 서버는 DNS 서버를 admin 서버로 지정
# NetworkManager에서 DNS 관리 끄기
cat << EOF > /etc/NetworkManager/conf.d/99-dns-none.conf
[main]
dns=none
EOF
systemctl restart NetworkManager

# DNS 서버 정보 설정
nmcli connection modify enp0s3 ipv4.dns "192.168.10.10"
nmcli connection up enp0s3
echo "nameserver 192.168.10.10" > /etc/resolv.conf

# 확인
dig +short google.com
172.217.211.101
172.217.211.138
172.217.211.139
172.217.211.100
172.217.211.102
172.217.211.113

 

Local (Mirror) YUM/DNF Repository 구성

  • kubespray-offline 설치시 자동으로 구성됨

Private Container (Image) Registry 구성

  • kubespray-offline 설치시 자동으로 구성됨

Private PyPI(Python Package Index) Mirror

  • kubespray-offline 설치시 자동으로 구성됨

kubespary offline 설치

kubespray는 오프라인 환경에서 k8s를 배포를 위한 편의성을 지원합니다.

 

kubespray/contrib/offline at master · kubernetes-sigs/kubespray

Deploy a Production Ready Kubernetes Cluster. Contribute to kubernetes-sigs/kubespray development by creating an account on GitHub.

github.com

  • 다운로드될 파일 목록과 컨테이너 이미지 목록을 파일별로 생성
  • 오프라인 배포를 위한 컨테이너 이미지 다운로드 및 이미지 레지스트리(저장소)에 등록(업로드)
  • 파일(목록)을 다운로드 하고 Nginx 컨테이너를 실행하여, 파일 다운로드 기능 제공

폐쇄망(Air-Gap) 환경에서 구성을 위한 지원

① generate_list 스크립트 실행

  • 인터넷이 되는 admin 서버에서 설치시 필요한 모든 파일과, 컨테이너 이미지를 다운로드

② manage-offline-files

  • dnf 레포지토리, pypi 레포지토리를 만들고 각 노드에서 업로드까지 완료해 줌

③ manage-offline-container-images

  • 내부 레지스트리 서버를 실행하고, 설치에 필요한 이미지를 내부 레지스트리 서버에 PUSH 수행

하지만 kubespray가 제공하는 스크립트의 기능은 다소 부족한 부분이 존재하여 tmurakam 이라는 엔지니어가 편리성을 추가한 github을 운영하고 있습니다. 이번 실습에서 이용할 예정입니다.

 

GitHub - kubespray-offline/kubespray-offline: kubespray offline support scripts

kubespray offline support scripts. Contribute to kubespray-offline/kubespray-offline development by creating an account on GitHub.

github.com

 

kubespary-offline 설치 실습

ⓞ git clone 후 download-all.sh 로 설치에 필요한 파일들 다운로드 수행 (3.3GB 정도)

  • config.sh 에 설정된 변수를 끝까지 사용함 --> 필요에 의해 재정의
# git clone
git clone https://github.com/kubespray-offline/kubespray-offline
cd kubespray-offline/

# 변수 정보 확인 : nginx 와 registry 는 각각 1.29.4 와 3.0.0 변수 선언 확인 <- 버전 변경 시에는 이 단계에서 수정 필요!
source ./config.sh
echo -e "kubespary $KUBESPRAY_VERSION"
echo -e "runc $RUNC_VERSION"
echo -e "containerd $CONTAINERD_VERSION"
echo -e "nercdtl $NERDCTL_VERSION"
echo -e "cni $CNI_VERSION"
echo -e "nginx $NGINX_VERSION"
echo -e "registry $REGISTRY_VERSION"
echo -e "registry_port: $REGISTRY_PORT"
echo -e "Additional container registry hosts: $ADDITIONAL_CONTAINER_REGISTRY_LIST"
echo -e "cpu arch: $IMAGE_ARCH"
kubespary 2.30.0
runc 1.3.4
containerd 2.2.1
nercdtl 2.2.1
cni 1.8.0
nginx 1.29.4
registry 3.0.0
registry_port: 35000
Additional container registry hosts: myregistry.io
cpu arch: amd64

# download-all.sh  실행 --> 17분 소요
./download-all.sh
...
(230/230): container-selinux-2.240.0-1.el10.noarch.rpm                                    4.6 kB/s |  56 kB     00:12    
/bin/rm: cannot remove 'outputs/rpms/local/*.i686.rpm': No such file or directory # 32비트 RPM 파일이 애초에 없어서 지울 게 없음. 오류 아니여서 무시해도됨.
==> createrepo
Directory walk started
Directory walk done - 230 packages
Temporary output repo path: outputs/rpms/local/.repodata/
Pool started (with 5 workers)
Pool finished
create-repo done. --> 내부 저장소 설치가 완료되었슴:패키지 저장소 + Pypi
=> Running: ./copy-target-scripts.sh
==> Copy target scripts
Done.

# 가상환경의 .venv 디렉터리 확인
du -sh ~/.venv
491M /root/.venv

tree ~/.venv | more
/root/.venv
└── 3.12
    ├── bin
    │   ├── activate
    │   ├── activate.csh
    │   ├── activate.fish
    │   ├── Activate.ps1
    │   ├── ansible
    ...

# /root/.cache 디렉터리 확인
tree ~/.cache | more
du -sh ~/.cache
819M /root/.cache

# 다운로드 될 파일과 이미지 생성 스크립트는 kubespary repo 에 offline 참고 확인
tree /root/kubespray-offline/cache/kubespray-2.30.0/contrib/offline/
/root/kubespray-offline/cache/kubespray-2.30.0/contrib/offline/
├── docker-daemon.json
├── generate_list.sh
├── generate_list.yml
├── manage-offline-container-images.sh
├── manage-offline-files.sh
├── nginx.conf
├── README.md
├── registries.conf
├── temp
│   ├── files.list
│   ├── files.list.template
│   ├── images.list
│   └── images.list.template
└── upload2artifactory.py

2 directories, 13 files

# 용량 확인
du -sh /root/kubespray-offline/outputs/
3.3G    outputs/

# 디렉터리/파일 구조 확인
tree /root/kubespray-offline/outputs/ | more
tree /root/kubespray-offline/outputs/ -L 1
outputs/
├── config.sh
├── config.toml
├── containerd.service
├── extract-kubespray.sh
├── files                  # kubectl/kubelet/kubeadm, containerd 등 바이너리 파일들
├── images                 # 컨테이너 이미지를 .tar.gz 압축파일
├── install-containerd.sh
├── load-push-all-images.sh
├── nginx-default.conf
├── patches
├── playbook               # 노드들에 offline repo 설정을 위한 playbook/role
├── pypi                   # python 패키지 파일들 - index.html, *.whl, *.tar.gz
├── pyver.sh
├── rpms                   # rpm 패키지 파일들
├── setup-all.sh
├── setup-container.sh
├── setup-offline.sh
├── setup-py.sh
├── start-nginx.sh
├── start-registry.sh
└── venv.sh

7 directories, 15 files

 

① outputs 디렉터리 이동 후 setup-container.sh 실행 :  추가로 install-containerd.sh 실행됨

  • containerd 설치
# outputs 디렉토리 확인 --> 후속 실행스크립트 존재함
cd /root/kubespray-offline/outputs
ls -l *.sh
-rw-r--r--. 1 root root 1371 Feb  3 19:46 config.sh
-rwxr-xr-x. 1 root root  719 Feb  3 19:46 extract-kubespray.sh
-rwxr-xr-x. 1 root root 2544 Feb  3 19:46 install-containerd.sh
-rwxr-xr-x. 1 root root 1141 Feb  3 19:46 load-push-all-images.sh
-rw-r--r--. 1 root root  607 Feb  3 19:46 pyver.sh
-rwxr-xr-x. 1 root root  394 Feb  3 19:46 setup-all.sh
-rwxr-xr-x. 1 root root  408 Feb  3 19:46 setup-container.sh
-rwxr-xr-x. 1 root root 2106 Feb  3 19:46 setup-offline.sh
-rwxr-xr-x. 1 root root 1213 Feb  3 19:46 setup-py.sh
-rwxr-xr-x. 1 root root  654 Feb  3 19:46 start-nginx.sh
-rwxr-xr-x. 1 root root  445 Feb  3 19:46 start-registry.sh
-rw-r--r--. 1 root root  322 Feb  3 19:46 venv.sh

# Install containerd from local files. Load nginx and registry images to containerd.
./setup-container.sh
==> Install runc
==> Install nerdctl
==> Install containerd
==> Start containerd
==> Install CNI plugins
==> Load registry, nginx images

# 설치된 바이너리 파일 및 버전 확인
which runc && runc --version
/usr/local/bin/runc
runc version 1.3.4
commit: v1.3.4-0-gd6d73eb8
spec: 1.2.1
go: go1.24.10
libseccomp: 2.5.6

which containerd && containerd --version
/usr/local/bin/containerd
containerd github.com/containerd/containerd/v2 v2.2.1 dea7da592f5d1d2b7755e3a161be07f43fad8f75

which nerdctl && nerdctl --version
/usr/local/bin/nerdctl
nerdctl version 2.2.1

tree -ug /opt/cni/bin/
[root     root    ]  /opt/cni/bin/
├── [root     root    ]  bandwidth
├── [root     root    ]  bridge
├── [root     root    ]  dhcp
├── [root     root    ]  dummy
├── [root     root    ]  firewall
├── [root     root    ]  host-device
├── [root     root    ]  host-local
├── [root     root    ]  ipvlan
├── [root     root    ]  LICENSE
├── [root     root    ]  loopback
├── [root     root    ]  macvlan
├── [root     root    ]  portmap
├── [root     root    ]  ptp
├── [root     root    ]  README.md
├── [root     root    ]  sbr
├── [root     root    ]  static
├── [root     root    ]  tap
├── [root     root    ]  tuning
├── [root     root    ]  vlan
└── [root     root    ]  vrf

# containerd systemd unit 확인
cat /etc/containerd/config.toml
version = 2
root = "/var/lib/containerd"
state = "/run/containerd"
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  uid = 0
  gid = 0

[debug]
  address = "/run/containerd/debug.sock"
  uid = 0
  gid = 0
  level = "info"

[metrics]
  address = ""
  grpc_histogram = false

[cgroup]
  path = ""

[plugins]
  [plugins."io.containerd.grpc.v1.cri".containerd]
    default_runtime_name = "runc"
    snapshotter = "overlayfs"
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
    runtime_type = "io.containerd.runc.v2"
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    systemdCgroup = true

cat /etc/systemd/system/containerd.service
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd

Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target

systemctl status containerd.service --no-pager
● containerd.service - containerd container runtime
     Loaded: loaded (/etc/systemd/system/containerd.service; enabled; preset: disabled)
     Active: active (running) since Sat 2026-02-14 14:02:52 KST; 2min 40s ago
 Invocation: abe3bfcee0d24001b9765d1555e08206
       Docs: https://containerd.io
    Process: 20522 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
   Main PID: 20524 (containerd)
      Tasks: 10
     Memory: 516.2M (peak: 519.2M)
        CPU: 6.610s
     CGroup: /system.slice/containerd.service
             └─20524 /usr/local/bin/containerd

# 다운받은 이미지를 압축해제 후 로컬에 load 확인 : CPU Arch = PLATFORM 확인!
nerdctl images
REPOSITORY    TAG              IMAGE ID        CREATED          PLATFORM       SIZE       BLOB SIZE
nginx         1.29.4           93c49ce72e03    2 minutes ago    linux/amd64    171MB      164.3MB
nginx         1.28.0-alpine    dc8e6d3967a0    3 minutes ago    linux/amd64    51.18MB    49.69MB
registry      3.0.0            09d6d68c85b9    3 minutes ago    linux/amd64    58.44MB    58.26MB
registry      2.8.1            1e6c7d1be0dd    3 minutes ago    linux/amd64    26.65MB    26.49MB

 

② start-nginx.sh 실행 : 웹 서버로 files, images, pypi, rpms 제공

# (옵션) nginx conf 파일 수정 : 디렉터리 목록 표시되게
cp nginx-default.conf nginx-default.bak
cat << EOF > nginx-default.conf 
server {
    listen       80;
    listen  [::]:80;
    server_name  localhost;

    location / {
        root   /usr/share/nginx/html;
        # index  index.html index.htm;

        autoindex on;                 # 디렉터리 목록 표시
        autoindex_exact_size off;     # 파일 크기 KB/MB/GB 단위로 보기 좋게
        autoindex_localtime on;       # 서버 로컬 타임으로 표시
    }

    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

    # Force sendfile to off
    sendfile off;     
}
EOF

# Start nginx container.
./start-nginx.sh
===> Stop nginx
===> Start nginx
3a6f7cb1df6bfac6461aaafaf9a6bfa9db6fb38fabbfe5cf592a3b93569bf1dc

# nginx 컨테이너 확인
nerdctl ps
CONTAINER ID    IMAGE                             COMMAND                   CREATED               STATUS    PORTS    NAMES
3a6f7cb1df6b    docker.io/library/nginx:1.29.4    "/docker-entrypoint.…"    About a minute ago    Up                 nginx

# nginx 웹 접속
open http://192.168.10.10

 

③ setup-offline.sh 실행 : offline repo 설정, pypi mirror 전역 설정

# 스크립트 실행 전 기본 정보 확인
dnf repolist
repo id                                              repo name
appstream                                            Rocky Linux 10 - AppStream
baseos                                               Rocky Linux 10 - BaseOS
extras                                               Rocky Linux 10 - Extras

cat /etc/redhat-release
Rocky Linux release 10.0 (Red Quartz)


# 스크립트 실행 : Setup yum/deb repo config and PyPI mirror config to use local nginx server.
./setup-offline.sh
/bin/rm: cannot remove '/etc/yum.repos.d/offline.repo': No such file or directory
===> Disable all yumrepositories
===> Setup local yum repository
[offline-repo]
name=Offline repo
baseurl=http://localhost/rpms/local/
enabled=1
gpgcheck=0
===> Setup PyPI mirror

# 기존 repo 이름이 .original로 변경되고, offline.repo 추가 확인
tree /etc/yum.repos.d/
/etc/yum.repos.d/
├── offline.repo
├── rocky-addons.repo.original
├── rocky-devel.repo.original
├── rocky-extras.repo.original
└── rocky.repo.original

# offline.repo 파일 확인
cat /etc/yum.repos.d/offline.repo
[offline-repo]
name=Offline repo
baseurl=http://localhost/rpms/local/
enabled=1
gpgcheck=0

dnf clean all
dnf repolist
repo id                                                      repo name
offline-repo                                                 Offline repo

# pip 전역 설정 : pypi mirror 설정 확인
cat ~/.config/pip/pip.conf
[global]
index = http://localhost/pypi/
index-url = http://localhost/pypi/
trusted-host = localhost

 

④ setup-py.sh 실행 : offline repo 로 부터 python${PY} 설치 시도

# Install python3 and venv from local repo.
## sudo dnf install -y --disablerepo=* --enablerepo=offline-repo python${PY}
./setup-py.sh
===> Install python, venv, etc
Last metadata expiration check: 0:06:40 ago on Wed 04 Feb 2026 12:23:11 AM KST.
Package python3-3.12.12-3.el10_1.aarch64 is already installed.
Dependencies resolved.
Nothing to do.
Complete!

# 변수 확인
source pyver.sh
echo -e "python_version $python${PY}"
python_version 3.12

# offline-repo 에 패키지 파일 확인
dnf info python3
Last metadata expiration check: 0:00:23 ago on Sat 14 Feb 2026 02:38:25 PM KST.
Installed Packages
Name         : python3
Version      : 3.12.12
Release      : 3.el10_1
Architecture : x86_64
Size         : 31 k
Source       : python3.12-3.12.12-3.el10_1.src.rpm
Repository   : @System
From repo    : baseos
Summary      : Python 3.12 interpreter
URL          : https://www.python.org/
License      : Python-2.0.1
Description  : Python 3.12 is an accessible, high-level, dynamically typed, interpreted
             : programming language, designed with an emphasis on code readability.
             : It includes an extensive standard library, and has a vast ecosystem of
             : third-party libraries.
             :
             : The python3 package provides the "python3" executable: the reference
             : interpreter for the Python language, version 3.
             : The majority of its standard library is provided in the python3-libs package,
             : which should be installed automatically along with python3.
             : The remaining parts of the Python standard library are broken out into the
             : python3-tkinter and python3-test packages, which may need to be installed
             : separately.
             :
             : Documentation for Python is provided in the python3-docs package.
             :
             : Packages containing additional libraries for Python are generally named with
             : the "python3-" prefix.

tree rpms/local/ | grep -i python
├── libcap-ng-python3-0.8.4-6.el10.x86_64.rpm
├── python3-3.12.12-3.el10_1.x86_64.rpm
├── python3-dateutil-2.9.0.post0-1.el10_0.noarch.rpm
├── python3-dbus-1.3.2-8.el10.x86_64.rpm
├── python3-devel-3.12.12-3.el10_1.x86_64.rpm
├── python3-dnf-4.20.0-18.el10.rocky.0.1.noarch.rpm
├── python3-dnf-plugins-core-4.7.0-9.el10.noarch.rpm
├── python3-dnf-plugin-versionlock-4.7.0-9.el10.noarch.rpm
├── python3-firewall-2.3.1-1.el10_0.noarch.rpm
├── python3-gobject-base-3.46.0-7.el10.x86_64.rpm
├── python3-gobject-base-noarch-3.46.0-7.el10.noarch.rpm
├── python3-hawkey-0.73.1-12.el10.rocky.0.1.x86_64.rpm
├── python3-libcomps-0.1.21-3.el10.x86_64.rpm
├── python3-libdnf-0.73.1-12.el10.rocky.0.1.x86_64.rpm
├── python3-libs-3.12.12-3.el10_1.x86_64.rpm
├── python3-libselinux-3.9-1.el10.x86_64.rpm
├── python3-nftables-1.1.1-6.el10.x86_64.rpm
├── python3-pip-23.3.2-7.el10.noarch.rpm
├── python3-pip-wheel-23.3.2-7.el10.noarch.rpm
├── python3-rpm-4.19.1.1-20.el10.x86_64.rpm
├── python3-six-1.16.0-16.el10.noarch.rpm
├── python3-systemd-235-11.el10.x86_64.rpm
├── python-unversioned-command-3.12.12-3.el10_1.noarch.rpm

 

⑤ start-registry.sh 실행 : (컨테이너) 이미지 저장소 컨테이너로 기동

# Start docker private registry container.
./start-registry.sh
===> Start registry

# 관련 변수 확인
source config.sh
echo -e "registry_port: $REGISTRY_PORT"
registry_port: 35000

# 확인
nerdctl ps
CONTAINER ID    IMAGE                               COMMAND                   CREATED           STATUS    PORTS    NAMES
348e5f4d5b13    docker.io/library/registry:3.0.0    "/entrypoint.sh /etc…"    27 seconds ago    Up                 registry
3a6f7cb1df6b    docker.io/library/nginx:1.29.4      "/docker-entrypoint.…"    34 minutes ago    Up                 nginx

ss -tnlp | grep registry
LISTEN 0      4096               *:35000            *:*    users:(("registry",pid=20993,fd=3))                                                 
LISTEN 0      4096               *:5001             *:*    users:(("registry",pid=20993,fd=6))   

# 현재는 registry 서버 내부에 저장된 (컨테이너) 이미지 없는 상태 : (참고) REGISTRY_DIR=${REGISTRY_DIR:-/var/lib/registry}
tree /var/lib/registry/
/var/lib/registry/
0 directories, 0 files

# tcp 5001 port : debug, metrics 
curl 192.168.10.10:5001/metrics
registry_storage_action_seconds_bucket{action="Stat",driver="filesystem",le="2.5"} 10
registry_storage_action_seconds_bucket{action="Stat",driver="filesystem",le="5"} 10
registry_storage_action_seconds_bucket{action="Stat",driver="filesystem",le="10"} 10
registry_storage_action_seconds_bucket{action="Stat",driver="filesystem",le="+Inf"} 10
registry_storage_action_seconds_sum{action="Stat",driver="filesystem"} 0.001790512
registry_storage_action_seconds_count{action="Stat",driver="filesystem"} 10

curl 192.168.10.10:5001/debug/pprof/

 

⑥ load-push-images.sh 실행 : (컨테이너) 이미지 저장소에 이미지 push

## (옵션) 'registry.k8s.io k8s.gcr.io gcr.io ghcr.io docker.io quay.io' 이외에 추가로 필요한 저장소가 있다면 설정
echo -e "Additional container registry hosts: $ADDITIONAL_CONTAINER_REGISTRY_LIST"
Additional container registry hosts: myregistry.io

# nerdctl load 할 .tar.gz 파일들
ls images/*.tar.gz | xargs -n 1 basename
docker.io_amazon_aws-alb-ingress-controller-v1.1.9.tar.gz
docker.io_amazon_aws-ebs-csi-driver-v0.5.0.tar.gz
docker.io_cloudnativelabs_kube-router-v2.1.1.tar.gz
docker.io_flannel_flannel-cni-plugin-v1.7.1-flannel1.tar.gz
docker.io_flannel_flannel-v0.27.3.tar.gz
docker.io_kubeovn_kube-ovn-v1.12.21.tar.gz
docker.io_kubernetesui_dashboard-v2.7.0.tar.gz
docker.io_kubernetesui_metrics-scraper-v1.0.8.tar.gz
docker.io_library_haproxy-3.2.4-alpine.tar.gz
docker.io_library_nginx-1.28.0-alpine.tar.gz
docker.io_library_nginx-1.29.4.tar.gz
docker.io_library_registry-2.8.1.tar.gz
docker.io_library_registry-3.0.0.tar.gz
docker.io_mirantis_k8s-netchecker-agent-v1.2.2.tar.gz
docker.io_mirantis_k8s-netchecker-server-v1.2.2.tar.gz
docker.io_rancher_local-path-provisioner-v0.0.32.tar.gz
ghcr.io_k8snetworkplumbingwg_multus-cni-v4.2.2.tar.gz
ghcr.io_kube-vip_kube-vip-v1.0.3.tar.gz
quay.io_calico_apiserver-v3.30.6.tar.gz
quay.io_calico_cni-v3.30.6.tar.gz
quay.io_calico_kube-controllers-v3.30.6.tar.gz
quay.io_calico_node-v3.30.6.tar.gz
quay.io_calico_typha-v3.30.6.tar.gz
quay.io_cilium_certgen-v0.2.4.tar.gz
quay.io_cilium_cilium-envoy-v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5.tar.gz
quay.io_cilium_cilium-v1.18.6.tar.gz
quay.io_cilium_hubble-relay-v1.18.6.tar.gz
quay.io_cilium_hubble-ui-backend-v0.13.3.tar.gz
quay.io_cilium_hubble-ui-v0.13.3.tar.gz
quay.io_cilium_operator-v1.18.6.tar.gz
quay.io_coreos_etcd-v3.5.26.tar.gz
quay.io_jetstack_cert-manager-cainjector-v1.15.3.tar.gz
quay.io_jetstack_cert-manager-controller-v1.15.3.tar.gz
quay.io_jetstack_cert-manager-webhook-v1.15.3.tar.gz
quay.io_metallb_controller-v0.13.9.tar.gz
quay.io_metallb_speaker-v0.13.9.tar.gz
registry.k8s.io_coredns_coredns-v1.12.1.tar.gz
registry.k8s.io_cpa_cluster-proportional-autoscaler-v1.8.8.tar.gz
registry.k8s.io_dns_k8s-dns-node-cache-1.25.0.tar.gz
registry.k8s.io_ingress-nginx_controller-v1.13.3.tar.gz
registry.k8s.io_kube-apiserver-v1.34.3.tar.gz
registry.k8s.io_kube-controller-manager-v1.34.3.tar.gz
registry.k8s.io_kube-proxy-v1.34.3.tar.gz
registry.k8s.io_kube-scheduler-v1.34.3.tar.gz
registry.k8s.io_metrics-server_metrics-server-v0.8.0.tar.gz
registry.k8s.io_pause-3.10.1.tar.gz
registry.k8s.io_provider-os_cinder-csi-plugin-v1.30.0.tar.gz
registry.k8s.io_sig-storage_csi-attacher-v4.4.2.tar.gz
registry.k8s.io_sig-storage_csi-node-driver-registrar-v2.4.0.tar.gz
registry.k8s.io_sig-storage_csi-provisioner-v3.6.2.tar.gz
registry.k8s.io_sig-storage_csi-resizer-v1.9.2.tar.gz
registry.k8s.io_sig-storage_csi-snapshotter-v6.3.2.tar.gz
registry.k8s.io_sig-storage_livenessprobe-v2.11.0.tar.gz
registry.k8s.io_sig-storage_local-volume-provisioner-v2.5.0.tar.gz
registry.k8s.io_sig-storage_snapshot-controller-v7.0.2.tar.gz

# Load all container images to containerd. Tag and push them to the private registry.
# x86 CPU 에서는 에러없이 진행됨
./load-push-all-images.sh

# (트러블슈팅) mac사용자: 아래 내용 추가 후 스크립트 다시 실행
vi load-push-all-images.sh
# 아래 추가 -----------------------
...
load_images() {
    for image in $BASEDIR/images/*.tar.gz; do
        echo "===> Loading $image"
        sudo $NERDCTL load --all-platforms -i $image || exit 1
    done
...
push_images() {
    ...
        echo "===> Push ${newImage}"
        sudo $NERDCTL push --platform=linux/arm64 ${newImage} || exit 1 # 왼쪽 추가 안해도 됨
    done
# -------------------------------


# 로컬 이미지 load 확인
nerdctl images
REPOSITORY                                               TAG
localhost:35000/kube-proxy                               v1.34.3
localhost:35000/kube-scheduler                           v1.34.3
localhost:35000/kube-controller-manager                  v1.34.3
localhost:35000/kube-apiserver                           v1.34.3
localhost:35000/metallb/controller                       v0.13.9
localhost:35000/metallb/speaker                          v0.13.9
localhost:35000/kubernetesui/metrics-scraper             v1.0.8
localhost:35000/kubernetesui/dashboard                   v2.7.0
localhost:35000/amazon/aws-ebs-csi-driver                v0.5.0
localhost:35000/provider-os/cinder-csi-plugin            v1.30.0
localhost:35000/sig-storage/csi-node-driver-registrar    v2.4.0
localhost:35000/sig-storage/livenessprobe                v2.11.0
localhost:35000/sig-storage/csi-resizer                  v1.9.2
localhost:35000/sig-storage/snapshot-controller          v7.0.2
localhost:35000/sig-storage/csi-snapshotter              v6.3.2
localhost:35000/sig-storage/csi-provisioner              v3.6.2
localhost:35000/sig-storage/csi-attacher                 v4.4.2
localhost:35000/jetstack/cert-manager-webhook            v1.15.3
localhost:35000/jetstack/cert-manager-cainjector         v1.15.3
localhost:35000/jetstack/cert-manager-controller         v1.15.3
localhost:35000/amazon/aws-alb-ingress-controller        v1.1.9
localhost:35000/ingress-nginx/controller                 v1.13.3
localhost:35000/rancher/local-path-provisioner           v0.0.32
localhost:35000/sig-storage/local-volume-provisioner     v2.5.0
localhost:35000/metrics-server/metrics-server            v0.8.0
localhost:35000/library/registry                         2.8.1
localhost:35000/cpa/cluster-proportional-autoscaler      v1.8.8
localhost:35000/dns/k8s-dns-node-cache                   1.25.0
localhost:35000/coredns/coredns                          v1.12.1
localhost:35000/library/haproxy                          3.2.4-alpine
localhost:35000/library/nginx                            1.28.0-alpine
localhost:35000/kube-vip/kube-vip                        v1.0.3
localhost:35000/pause                                    3.10.1
localhost:35000/cloudnativelabs/kube-router              v2.1.1
localhost:35000/kubeovn/kube-ovn                         v1.12.21
localhost:35000/calico/apiserver                         v3.30.6
localhost:35000/calico/typha                             v3.30.6
localhost:35000/calico/kube-controllers                  v3.30.6
localhost:35000/calico/cni                               v3.30.6
localhost:35000/calico/node                              v3.30.6
localhost:35000/flannel/flannel-cni-plugin               v1.7.1-flannel1
localhost:35000/flannel/flannel                          v0.27.3
localhost:35000/k8snetworkplumbingwg/multus-cni          v4.2.2
localhost:35000/cilium/cilium-envoy                      v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5
localhost:35000/cilium/hubble-ui-backend                 v0.13.3
localhost:35000/cilium/hubble-ui                         v0.13.3
localhost:35000/cilium/certgen                           v0.2.4
localhost:35000/cilium/hubble-relay                      v1.18.6
localhost:35000/cilium/operator                          v1.18.6
localhost:35000/cilium/cilium                            v1.18.6
localhost:35000/coreos/etcd                              v3.5.26
localhost:35000/mirantis/k8s-netchecker-agent            v1.2.2
localhost:35000/mirantis/k8s-netchecker-server           v1.2.2
localhost:35000/library/registry                         3.0.0
localhost:35000/library/nginx                            1.29.4
registry.k8s.io/sig-storage/snapshot-controller          v7.0.2
registry.k8s.io/sig-storage/local-volume-provisioner     v2.5.0
registry.k8s.io/sig-storage/livenessprobe                v2.11.0
registry.k8s.io/sig-storage/csi-snapshotter              v6.3.2
registry.k8s.io/sig-storage/csi-resizer                  v1.9.2
registry.k8s.io/sig-storage/csi-provisioner              v3.6.2
registry.k8s.io/sig-storage/csi-node-driver-registrar    v2.4.0
registry.k8s.io/sig-storage/csi-attacher                 v4.4.2
registry.k8s.io/provider-os/cinder-csi-plugin            v1.30.0
registry.k8s.io/pause                                    3.10.1
registry.k8s.io/metrics-server/metrics-server            v0.8.0
registry.k8s.io/kube-scheduler                           v1.34.3
registry.k8s.io/kube-proxy                               v1.34.3
registry.k8s.io/kube-controller-manager                  v1.34.3
registry.k8s.io/kube-apiserver                           v1.34.3
registry.k8s.io/ingress-nginx/controller                 v1.13.3
registry.k8s.io/dns/k8s-dns-node-cache                   1.25.0
registry.k8s.io/cpa/cluster-proportional-autoscaler      v1.8.8
registry.k8s.io/coredns/coredns                          v1.12.1
quay.io/metallb/speaker                                  v0.13.9
quay.io/metallb/controller                               v0.13.9
quay.io/jetstack/cert-manager-webhook                    v1.15.3
quay.io/jetstack/cert-manager-controller                 v1.15.3
quay.io/jetstack/cert-manager-cainjector                 v1.15.3
quay.io/coreos/etcd                                      v3.5.26
quay.io/cilium/operator                                  v1.18.6
quay.io/cilium/hubble-ui                                 v0.13.3
quay.io/cilium/hubble-ui-backend                         v0.13.3
quay.io/cilium/hubble-relay                              v1.18.6
quay.io/cilium/cilium                                    v1.18.6
quay.io/cilium/cilium-envoy                              v1.34.10-1762597008-ff7ae7d623be00078865cff1b0672cc5d9bfc6d5
quay.io/cilium/certgen                                   v0.2.4
quay.io/calico/typha                                     v3.30.6
quay.io/calico/node                                      v3.30.6
quay.io/calico/kube-controllers                          v3.30.6
quay.io/calico/cni                                       v3.30.6
quay.io/calico/apiserver                                 v3.30.6
ghcr.io/kube-vip/kube-vip                                v1.0.3
ghcr.io/k8snetworkplumbingwg/multus-cni                  v4.2.2
rancher/local-path-provisioner                           v0.0.32
mirantis/k8s-netchecker-server                           v1.2.2
mirantis/k8s-netchecker-agent                            v1.2.2
haproxy                                                  3.2.4-alpine
kubernetesui/metrics-scraper                             v1.0.8
kubernetesui/dashboard                                   v2.7.0
kubeovn/kube-ovn                                         v1.12.21
flannel/flannel                                          v0.27.3
flannel/flannel-cni-plugin                               v1.7.1-flannel1
cloudnativelabs/kube-router                              v2.1.1
amazon/aws-ebs-csi-driver                                v0.5.0
amazon/aws-alb-ingress-controller                        v1.1.9
nginx                                                    1.29.4
nginx                                                    1.28.0-alpine
registry                                                 3.0.0
registry                                                 2.8.1


# (참고) kube-proxy 컨테이너가 localhost:35000 과 registry.k8s.io 로 push 된 상태 확인
nerdctl images | grep -i kube-proxy
localhost:35000/kube-proxy  v1.34.3  fbe99026b627    3 minutes ago  linux/amd64    75.24MB    73.14MB
registry.k8s.io/kube-proxy  v1.34.3  fbe99026b627    7 minutes ago  linux/amd64    75.24MB    73.14MB

# localhost 있는 이미지가 각각 registry, quay, rancher, flannel 등으로 push 된것을 알 수 있다
nerdctl images | grep localhost | wc -l
55

nerdctl images | grep -v localhost | wc -l
56

# 이미지 저장소 카탈로그 확인
curl -s http://localhost:35000/v2/_catalog | jq
{
  "repositories": [
    "amazon/aws-alb-ingress-controller",
    "amazon/aws-ebs-csi-driver",
    ...
 
# kube-apiserver 정보 확인  
curl -s http://localhost:35000/v2/kube-apiserver/tags/list | jq
{
  "name": "kube-apiserver",
  "tags": [
    "v1.34.3"
  ]
}

## Image Manifest
curl -s http://localhost:35000/v2/kube-apiserver/manifests/v1.34.3 | jq
{
  "schemaVersion": 2,
  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
  "config": {
    "mediaType": "application/vnd.docker.container.image.v1+json",
    "digest": "sha256:cf65ae6c8f700cc27f57b7305c6e2b71276a7eed943c559a0091e1e667169896",
    "size": 2906
  },
  "layers": [
    {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar",
      "digest": "sha256:378b3db0974f7a5a8767b6329ad310983bc712d0e400ff5faa294f95f869cc8c",
      "size": 327680
    },
  ...(생략)

# 이미지 저장소의 저장 디렉터리 확인
tree /var/lib/registry/ -L 5
/var/lib/registry/
└── docker
    └── registry
        └── v2
            ├── blobs
            │   └── sha256
            └── repositories
                ├── amazon
                ├── calico
                ├── cilium
                ├── cloudnativelabs
                ├── coredns
                ├── coreos
                ├── cpa
                ├── dns
                ├── flannel
                ├── ingress-nginx
                ├── jetstack
                ├── k8snetworkplumbingwg
                ├── kube-apiserver
                ├── kube-controller-manager
                ├── kubeovn
                ├── kube-proxy
                ├── kubernetesui
                ├── kube-scheduler
                ├── kube-vip
                ├── library
                ├── metallb
                ├── metrics-server
                ├── mirantis
                ├── pause
                ├── provider-os
                ├── rancher
                └── sig-storage

 

⑦ extract-kubespary.sh 실행 : kubespary 저장소 압축 해제

# 스크립트 실행 전 관련 파일 확인 : kubespary repo 압축된 파일
ls -lh files/kubespray-*
-rw-r--r--. 1 root root 2.5M Feb 14 12:48 files/kubespray-2.30.0.tar.gz

# patches 파일 : kubespary-2.18.0 버전에서 patch 되는 내용으로, kubespary-2.30.0과 관계없음
tree patches/
patches/
└── 2.18.0
    ├── 0001-nerdctl-insecure-registry-config-8339.patch
    ├── 0002-Update-config.toml.j2-8340.patch
    └── 0003-generate-list-8537.patch

2 directories, 3 files

# Extract kubespray tarball and apply all patches.
./extract-kubespray.sh

# kubespary 저장소 압축해제된 파일들 확인
tree kubespray-2.30.0/ -L 1
kubespray-2.30.0/
├── ansible.cfg
├── CHANGELOG.md
├── cluster.yml
├── CNAME
├── code-of-conduct.md
├── _config.yml
├── contrib
├── CONTRIBUTING.md
├── Dockerfile
├── docs
├── extra_playbooks
├── galaxy.yml
├── index.html
├── inventory
├── library
├── LICENSE
├── logo
├── meta
├── OWNERS
├── OWNERS_ALIASES
├── pipeline.Dockerfile
├── playbooks
├── plugins
├── README.md
├── recover-control-plane.yml
├── RELEASE.md
├── remove-node.yml
├── remove_node.yml
├── requirements.txt
├── reset.yml
├── roles
├── scale.yml
├── scripts
├── SECURITY_CONTACTS
├── test-infra
├── tests
├── upgrade-cluster.yml
├── upgrade_cluster.yml
└── Vagrantfile

14 directories, 26 files

 

kubespary 설치

# venv 실행
python3.12 -m venv ~/.venv/3.12
source ~/.venv/3.12/bin/activate
which ansible
/root/.venv/3.12/bin/ansible

tree ~/.venv/3.12/ -L 4
/root/.venv/3.12/
├── bin
│   ├── activate
│   ├── activate.csh
│   ├── activate.fish
│   ├── Activate.ps1
│   ├── ansible
...

# kubespary 디렉터리 이동
cd /root/kubespray-offline/outputs/kubespray-2.30.0

# Install ansible : 이미 설치 완료된 상태
# update pip
pip install -U pip
Looking in indexes: http://localhost/pypi/
Requirement already satisfied: pip in /root/.venv/3.12/lib64/python3.12/site-packages (26.0.1)

# Install ansible
pip install -r requirements.txt   

# offline.yml 파일 복사 후 inventory 복사
cp ../../offline.yml .
cp -r inventory/sample inventory/mycluster
tree inventory/mycluster/
inventory/mycluster/
├── group_vars
│   ├── all
│   │   ├── all.yml
│   │   ├── aws.yml
│   │   ├── azure.yml
│   │   ├── containerd.yml
│   │   ├── coreos.yml
│   │   ├── cri-o.yml
│   │   ├── docker.yml
│   │   ├── etcd.yml
│   │   ├── gcp.yml
│   │   ├── hcloud.yml
│   │   ├── huaweicloud.yml
│   │   ├── oci.yml
│   │   ├── offline.yml
│   │   ├── openstack.yml
│   │   ├── upcloud.yml
│   │   └── vsphere.yml
│   └── k8s_cluster
│       ├── addons.yml
│       ├── k8s-cluster.yml
│       ├── k8s-net-calico.yml
│       ├── k8s-net-cilium.yml
│       ├── k8s-net-custom-cni.yml
│       ├── k8s-net-flannel.yml
│       ├── k8s-net-kube-ovn.yml
│       ├── k8s-net-kube-router.yml
│       ├── k8s-net-macvlan.yml
│       └── kube_control_plane.yml
└── inventory.ini

4 directories, 27 files

# 웹서버와 이미지 저장소 정보 수정 : http_server, registry_host
sed -i "s/YOUR_HOST/192.168.10.10/g" offline.yml
cat offline.yml | grep 192.168.10.10
http_server: "http://192.168.10.10"
registry_host: "192.168.10.10:35000"

# 수정 반영된 offline.yml 파일을 inventory 디렉터리 내부로 복사
\cp -f offline.yml inventory/mycluster/group_vars/all/offline.yml

# inventory 파일 작성
cat <<EOF > inventory/mycluster/inventory.ini
[kube_control_plane]
k8s-node1 ansible_host=192.168.10.11 ip=192.168.10.11 etcd_member_name=etcd1

[etcd:children]
kube_control_plane

[kube_node]
k8s-node2 ansible_host=192.168.10.12 ip=192.168.10.12
EOF

# ansible 연결 확인
ansible -i inventory/mycluster/inventory.ini all -m ping
[WARNING]: Platform linux on host k8s-node1 is using the discovered Python interpreter at /usr/bin/python3.12, but future installation of another Python interpreter could change the
meaning of that path. See https://docs.ansible.com/ansible-core/2.17/reference_appendices/interpreter_discovery.html for more information.
k8s-node1 | SUCCESS => {
    "ansible_facts": {
        "discovered_interpreter_python": "/usr/bin/python3.12"
    },
    "changed": false,
    "ping": "pong"
}
[WARNING]: Platform linux on host k8s-node2 is using the discovered Python interpreter at /usr/bin/python3.12, but future installation of another Python interpreter could change the
meaning of that path. See https://docs.ansible.com/ansible-core/2.17/reference_appendices/interpreter_discovery.html for more information.
k8s-node2 | SUCCESS => {
    "ansible_facts": {
        "discovered_interpreter_python": "/usr/bin/python3.12"
    },
    "changed": false,
    "ping": "pong"
}

# 각 노드에 offline repo 설정
tree ../playbook/
├── offline-repo.yml
└── roles
    └── offline-repo
        ├── defaults
        │   └── main.yml
        ├── files
        │   └── 99offline
        └── tasks
            ├── Debian.yml
            ├── main.yml
            └── RedHat.yml

mkdir offline-repo
cp -r ../playbook/ offline-repo/
ansible-playbook -i inventory/mycluster/inventory.ini offline-repo/playbook/offline-repo.yml

# k8s-node1 에 dnf 레포지토리가 설정되었는지 확인
ssh k8s-node1 tree /etc/yum.repos.d/
/etc/yum.repos.d/
├── offline.repo
├── rocky-addons.repo
├── rocky-devel.repo
├── rocky-extras.repo
└── rocky.repo

ssh k8s-node1 dnf repolist
repo id                          repo name
appstream                        Rocky Linux 10 - AppStream
baseos                           Rocky Linux 10 - BaseOS
extras                           Rocky Linux 10 - Extras
offline-repo                     Offline repo for kubespray

## 추가로 설치를 위해 기존 repo 제거 : 미실행할 경우, kubespary 실행 시 fail됨
for i in rocky-addons rocky-devel rocky-extras rocky; do
  ssh k8s-node1 "mv /etc/yum.repos.d/$i.repo /etc/yum.repos.d/$i.repo.original"
  ssh k8s-node2 "mv /etc/yum.repos.d/$i.repo /etc/yum.repos.d/$i.repo.original"
done

ssh k8s-node1 dnf repolist
repo id                          repo name
offline-repo                     Offline repo for kubespray

ssh k8s-node2 dnf repolist
repo id                          repo name
offline-repo                     Offline repo for kubespray

# admin-lb 에 kubectl 없는 것 확인
which kubectl
/usr/bin/which: no kubectl in (/root/.venv/3.12/bin:/root/.local/bin:/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin)

# group vars 실습 환경에 맞게 설정
echo "kubectl_localhost: true" >> inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml # 배포를 수행하는 로컬 머신의 bin 디렉토리에도 kubectl 바이너리를 다운로드
sed -i 's|kube_owner: kube|kube_owner: root|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|kube_network_plugin: calico|kube_network_plugin: flannel|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|kube_proxy_mode: ipvs|kube_proxy_mode: iptables|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|enable_nodelocaldns: true|enable_nodelocaldns: false|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
grep -iE 'kube_owner|kube_network_plugin:|kube_proxy_mode|enable_nodelocaldns:' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
echo "enable_dns_autoscaler: false" >> inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml

echo "flannel_interface: enp0s8" >> inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml
grep "^[^#]" inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml

sed -i 's|helm_enabled: false|helm_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
sed -i 's|metrics_server_enabled: false|metrics_server_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
grep -iE 'metrics_server_enabled:' inventory/mycluster/group_vars/k8s_cluster/addons.yml
echo "metrics_server_requests_cpu: 25m"     >> inventory/mycluster/group_vars/k8s_cluster/addons.yml
echo "metrics_server_requests_memory: 16Mi" >> inventory/mycluster/group_vars/k8s_cluster/addons.yml

kube_owner: root
kube_network_plugin: flannel
kube_proxy_mode: iptables
enable_nodelocaldns: false
flannel_interface: enp0s8
metrics_server_enabled: true

# 지원 버전 정보 확인
cat roles/kubespray_defaults/vars/main/checksums.yml | grep -i kube -A40


# [macOS 사용자] (TS) 이슈 해결
# -----------------------------------------------
TASK [download : Download_file | Download item] **************************************************************************
fatal: [k8s-node1]: FAILED! => {"attempts": 4, "changed": false, "dest": "/tmp/releases/etcd-3.5.26-linux-arm64.tar.gz", "elapsed": 0, "msg": "Request failed", "response": "HTTP Error 404: Not Found", "status_code": 404, "url": "http://192.168.10.10/files/kubernetes/etcd/etcd-v3.5.26-linux-amd64.tar.gz"}
http://192.168.10.10/files/kubernetes/etcd/etcd-v3.5.26-linux-amd64.tar.gz
# vi roles/download/tasks/download_file.yml >> no_log: false # (참고) 실패 task 상세 로그 출력 설정하여 원인 파악

cat inventory/mycluster/group_vars/all/offline.yml | grep amd64
etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-amd64.tar.gz"
sed -i 's/amd64/arm64/g' inventory/mycluster/group_vars/all/offline.yml
# -----------------------------------------------

# 배포 : 설치 완료까지 3분!
ansible-playbook -i inventory/mycluster/inventory.ini -v cluster.yml -e kube_version="1.34.3"


# 설치 후 NetworkManger 에 dns 설정 파일 추가 확인
ssh k8s-node2 cat /etc/NetworkManager/conf.d/dns.conf
[global-dns-domain-*]
servers = 10.233.0.3,192.168.10.10
[global-dns]
searches = default.svc.cluster.local,svc.cluster.local
options = ndots:2,timeout:2,attempts:2
# 하지만 '/etc/NetworkManager/conf.d/99-dns-none.conf' 파일로 인해, 위 설정이 resolv.conf 에 반영되지 않음, 즉 노드에서는 service명으로 도메인 질의는 불가능.
ssh k8s-node2 cat /etc/resolv.conf
nameserver 192.168.10.10

# 설치 후 NetworkManger 에서 특정 NIC은 관리하지 않게 설정 추가 확인
ssh k8s-node2 cat /etc/NetworkManager/conf.d/k8s.conf
[keyfile]
unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns

# kubectl 바이너리 파일을 ansible-playbook 실행한 서버에 다운로드 확인
ls -l inventory/mycluster/artifacts/kubectl
-rwxr-xr-x. 1 root root 60563640 Feb 14 15:46 inventory/mycluster/artifacts/kubectl

tree inventory/mycluster/
inventory/mycluster/
├── artifacts
│   └── kubectl
├── credentials
│   └── kubeadm_certificate_key.creds
├── group_vars
│   ├── all
│   │   ├── all.yml
│   │   ├── aws.yml
│   │   ├── azure.yml
│   │   ├── containerd.yml
│   │   ├── coreos.yml
│   │   ├── cri-o.yml
│   │   ├── docker.yml
│   │   ├── etcd.yml
│   │   ├── gcp.yml
│   │   ├── hcloud.yml
│   │   ├── huaweicloud.yml
│   │   ├── oci.yml
│   │   ├── offline.yml
│   │   ├── openstack.yml
│   │   ├── upcloud.yml
│   │   └── vsphere.yml
│   └── k8s_cluster
│       ├── addons.yml
│       ├── k8s-cluster.yml
│       ├── k8s-net-calico.yml
│       ├── k8s-net-cilium.yml
│       ├── k8s-net-custom-cni.yml
│       ├── k8s-net-flannel.yml
│       ├── k8s-net-kube-ovn.yml
│       ├── k8s-net-kube-router.yml
│       ├── k8s-net-macvlan.yml
│       └── kube_control_plane.yml
└── inventory.ini

6 directories, 29 files

cp inventory/mycluster/artifacts/kubectl /usr/local/bin/
kubectl version --client=true
Client Version: v1.34.3
Kustomize Version: v5.7.1

# k8s admin 자격증명 확인 
mkdir /root/.kube
scp k8s-node1:/root/.kube/config /root/.kube/
sed -i 's/127.0.0.1/192.168.10.11/g' /root/.kube/config
kubectl get no
NAME        STATUS   ROLES           AGE    VERSION
k8s-node1   Ready    control-plane   8m1s   v1.34.3
k8s-node2   Ready    <none>          7m1s   v1.34.3

# 자동완성 및 단축키 설정
source <(kubectl completion bash)
alias k=kubectl
complete -F __start_kubectl k
echo 'source <(kubectl completion bash)' >> /etc/profile
echo 'alias k=kubectl' >> /etc/profile
echo 'complete -F __start_kubectl k' >> /etc/profile

# 이미지 저장소가 192.168.10.10:35000 임을 확인
kubectl get deploy,sts,ds -n kube-system -owide
NAME                             READY   UP-TO-DATE   AVAILABLE   AGE     CONTAINERS       IMAGES                                                     SELECTOR
deployment.apps/coredns          2/2     2            2           7m14s   coredns          192.168.10.10:35000/coredns/coredns:v1.12.1                k8s-app=kube-dns
deployment.apps/metrics-server   1/1     1            1           6m31s   metrics-server   192.168.10.10:35000/metrics-server/metrics-server:v0.8.0   app.kubernetes.io/name=metrics-server,version=0.8.0

NAME                                     DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE     CONTAINERS     IMAGES                                        SELECTOR
daemonset.apps/kube-flannel              2         2         2       2            2           <none>                   7m43s   kube-flannel   192.168.10.10:35000/flannel/flannel:v0.27.3   app=flannel
daemonset.apps/kube-flannel-ds-arm       0         0         0       0            0           <none>                   7m42s   kube-flannel   192.168.10.10:35000/flannel/flannel:v0.27.3   app=flannel
daemonset.apps/kube-flannel-ds-arm64     0         0         0       0            0           <none>                   7m42s   kube-flannel   192.168.10.10:35000/flannel/flannel:v0.27.3   app=flannel
daemonset.apps/kube-flannel-ds-ppc64le   0         0         0       0            0           <none>                   7m42s   kube-flannel   192.168.10.10:35000/flannel/flannel:v0.27.3   app=flannel
daemonset.apps/kube-flannel-ds-s390x     0         0         0       0            0           <none>                   7m42s   kube-flannel   192.168.10.10:35000/flannel/flannel:v0.27.3   app=flannel
daemonset.apps/kube-proxy                2         2         2       2            2           kubernetes.io/os=linux   9m16s   kube-proxy     192.168.10.10:35000/kube-proxy:v1.34.3        k8s-app=kube-proxy

 

K8S 관련 폐쇄망 서비스 실습

k8s 클러스터는 구성되었지만 POD가 배포되는 k8s-node2 에는 인터넷이 안되는 환경이라 컨테이너 이미지를 다운로드 하지 못합니다.

폐쇄망에서 k8s 클러스터에 서비스를 운영하려면 아래와 같은 추가 구성요소가 필요합니다.

샘플 앱 배포 : nginx:alpine

실습을 진행하면 아시겠지만 nginx:alpine 이미지는 hub.docker.com 레지스트리 서버에서 다운로드하게 되는데 인터넷이 안되어 오류가 발생할 것입니다.

# [admin] nginx 디플로이먼트 배포 시도
cat << EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:alpine   # docker.io/library/nginx:alpine
          ports:
            - containerPort: 80
EOF

# 확인 : docker.io/library/nginx:alpine 이미지 가져오기 실패!
kubectl describe pod
Warning  Failed     10s   kubelet            Failed to pull image "nginx:alpine": failed to pull and unpack image "docker.io/library/nginx:alpine": failed to resolve image: failed to do request: Head "https://registry-1.docker.io/v2/library/nginx/manifests/alpine": dial tcp 34.196.49.8:443: i/o timeout
  Warning  Failed     10s   kubelet            Error: ErrImagePull
  Normal   BackOff    9s    kubelet            Back-off pulling image "nginx:alpine"
  Warning  Failed     9s    kubelet            Error: ImagePullBackOff

 

  • Private 레지스트리 서버에 nginx:alpine 이미지 PUSH
# 로컬에 nginx:alpine 다운로드
podman pull nginx:alpine
docker.io/library/nginx:alpine 선택

podman images | grep nginx
docker.io/library/nginx                                alpine                                                        b76de378d572  9 days ago     63.5 MB

# (컨테이너) 이미지 저장소에 이미지 push
podman tag nginx:alpine 192.168.10.10:35000/library/nginx:alpine

# 기본적으로 컨테이너 엔진들은 HTTPS를 요구합니다. 내부망에서 HTTP로 테스트하려면 Registry 주소를 '안전하지 않은 저장소'로 등록해야 합니다.
# (참고) registries.conf 는 containers-common 설정이라서, 'podman, skopeo, buildah' 등 전부 동일하게 적용됨.
cat <<EOF >> /etc/containers/registries.conf
[[registry]]
location = "192.168.10.10:35000"
insecure = true
EOF

# 프라이빗 레지스트리에 업로드 : 성공!
podman push 192.168.10.10:35000/library/nginx:alpine

# 업로드된 이미지와 태그 조회
curl -s 192.168.10.10:35000/v2/_catalog | jq
{
  "repositories": [
     ...
    "library/nginx",

curl -s 192.168.10.10:35000/v2/library/nginx/tags/list | jq
{
  "name": "library/nginx",
  "tags": [
    "1.28.0-alpine",
    "1.29.4",
    "alpine"
  ]
}

 

  • 샘플 앱 이미지 업데이트 후 배포 확인
# 디플로이먼트에 이미지 정보 업데이트
kubectl set image deployment/nginx nginx=192.168.10.10:35000/library/nginx:alpine
kubectl get deploy -owide
NAME    READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS   IMAGES                                     SELECTOR
nginx   0/1     1            0           12m   nginx        192.168.10.10:35000/library/nginx:alpine   app=nginx

# 현재 파드 상태
kubectl get pod
NAME                    READY   STATUS    RESTARTS   AGE
nginx-5ff7dd7b8-r75pp   1/1     Running   0          57s

 

하지만 위의 방법은 모든 이미지를 private 레지스트리 서버에 PUSH 해야하는 불편함이 존재합니다.

레지스트리 서버 미러를 설정하면 훨씬 더 깔끔하게 위의 문제를 해결 할 수 있습니다.

# 삭제 후 다시 디플로이먼트 배포
kubectl delete deployments.apps nginx

cat << EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:alpine   # docker.io/library/nginx:alpine
          ports:
            - containerPort: 80
EOF

# 현재 파드 상태는 ImagePullBackOff 상태입니다.
kubectl get pod
NAME                    READY   STATUS             RESTARTS   AGE
nginx-54fc99c8d-qj2dx   0/1     ImagePullBackOff   0          60s

[k8s-node1, k8s-node2] 에서 작업
# docker.io 대신 내부 이미지 레지스트리 주소 설정
mkdir -p /etc/containerd/certs.d/docker.io

cat <<EOF > /etc/containerd/certs.d/docker.io/hosts.toml
server = "https://docker.io"         # [HTTPS] 원본 registry 주소 , docker.io 를 대상으로 할 때 이 설정을 참조한다는 의미

[host."http://192.168.10.10:35000"]  # [HTTP] 내부 레지스트리를 미러로 지정 , docker.io 대신 실제 이미지를 가져올 내부 레지스트리 주소
  capabilities = ["pull", "resolve"] # "pull": 이미지 다운로드 허용 , "resolve": 태그 → 다이제스트 해석
  skip_verify = true                 # HTTPS 인증서 검증 스킵 (HTTP라서 사실상 의미 없는지 테스트 해보자)
EOF

systemctl restart containerd

# 이미지 가져오기 실행 후 확인 : k8s-nodes는 현재 외부 통신 불능 상태인데, 아래 처럼 docker.io 미러 설정되어서 가져오는 것을 확인!
nerdctl pull docker.io/library/nginx:alpine
crictl images | grep nginx
192.168.10.10:35000/library/nginx                   alpine              aea88c29b151e       25.7MB
docker.io/library/nginx                             alpine              aea88c29b151e       25.7MB


# 현재 파드 상태는 Running 상태로 변하여 정상 수행됩니다.
kubectl get pod
NAME                    READY   STATUS    RESTARTS   AGE
nginx-54fc99c8d-8jpqt   1/1     Running  0          12m

 

위와 같은 작업을 클러스터의 모든 노드에 설정해야하는 번거로움을 kubespray를 이용하면 아래와 같이 자동화 할 수 있습니다.

  • kubespary 에 containerd_registries_mirrors values 설정 후 적용
[admin]

# containerd registry 정보 확인
cat /root/kubespray-offline/outputs/kubespray-2.30.0/inventory/mycluster/group_vars/all/offline.yml | head -n 15
...
http_server: "http://192.168.10.10"
registry_host: "192.168.10.10:35000"

# Insecure registries for containerd
containerd_registries_mirrors:
  - prefix: "{{ registry_host }}"
    mirrors:
      - host: "http://{{ registry_host }}"
        capabilities: ["pull", "resolve"]
        skip_verify: true

# 수정
nano inventory/mycluster/group_vars/all/offline.yml
-------------------------------------------------
# Insecure registries for containerd
containerd_registries_mirrors:
  - prefix: "{{ registry_host }}"
    mirrors:
      - host: "http://{{ registry_host }}"
        capabilities: ["pull", "resolve"]
        skip_verify: true
  - prefix: "docker.io"
    mirrors:
      - host: "http://192.168.10.10:35000"
        capabilities: ["pull", "resolve"]
        skip_verify: false
  - prefix: "registry-1.docker.io"
    mirrors:
      - host: "http://192.168.10.10:35000"
        capabilities: ["pull", "resolve"]
        skip_verify: false
  - prefix: "quay.io"
    mirrors:
      - host: "http://192.168.10.10:35000"
        capabilities: ["pull", "resolve"]
        skip_verify: false
-------------------------------------------------

# 설정 업데이트
ansible-playbook -i inventory/mycluster/inventory.ini -v cluster.yml -e kube_version="1.34.3" --tags containerd

# 확인
ssh k8s-node2 tree /etc/containerd
/etc/containerd
├── certs.d
│   ├── 192.168.10.10:35000
│   │   └── hosts.toml
│   ├── docker.io
│   │   └── hosts.toml
│   ├── quay.io
│   │   └── hosts.toml
│   └── registry-1.docker.io
│       └── hosts.toml
├── config.toml
└── cri-base.json

ssh k8s-node2 cat /etc/containerd/certs.d/quay.io/hosts.toml
server = "https://quay.io"
[host."http://192.168.10.10:35000"]
  capabilities = ["pull","resolve"]
  skip_verify = false
  override_path = false
(참고) 주요 이미지 저장소 미러 설정
더보기

# Insecure registries for containerd
containerd_registries_mirrors:
  - prefix: "{{ registry_host }}"
    mirrors:
      - host: "http://{{ registry_host }}"
        capabilities: ["pull", "resolve"]
        skip_verify: true
  - prefix: docker.io
    mirrors:
      - host: "http://{{ registry_host }}"
        capabilities: ["pull", "resolve"]
        skip_verify: true
  - prefix: registry-1.docker.io
    mirrors:
      - host: "http://192.168.10.10:35000"
        capabilities: ["pull", "resolve"]
        skip_verify: false
  - prefix: ghcr.io
    mirrors:
      - host: "http://{{ registry_host }}"
        capabilities: ["pull", "resolve"]
        skip_verify: true
  - prefix: gcr.io
    mirrors:
      - host: "http://{{ registry_host }}"
        capabilities: ["pull", "resolve"]
        skip_verify: true
  - prefix: quay.io
    mirrors:
      - host: "http://{{ registry_host }}"
        capabilities: ["pull", "resolve"]
        skip_verify: true
  - prefix: k8s.gcr.io
    mirrors:
      - host: "http://{{ registry_host }}" 
        capabilities: ["pull", "resolve"]
        skip_verify: true
  - prefix: registry.k8s.io
    mirrors:
      - host: "http://{{ registry_host }}" 
        capabilities: ["pull", "resolve"]
        skip_verify: true

 

'Kubernetes' 카테고리의 다른 글

[K8S Deploy 4주차] kubespray를 이용한 K8S 클러스터 구성  (0) 2026.01.31
[K8S Deploy 3주차] kubeadm 을 이용한 K8S 클러스터 구성  (0) 2026.01.22
[K8S Deploy 2주차] Ansible 기초  (0) 2026.01.12
[K8S Deploy 1주차] Kubernetes 손 설치  (1) 2026.01.10
MinIO - DirectPV & Performance  (1) 2025.09.20

'Kubernetes'의 다른글

  • 현재글[K8S Deploy 6주차] 폐쇄망에서 kubespray를 이용한 K8S 구성

관련글

티스토리툴바