cloud-computing-msc-ai-examples/terraform/K3S/insecure.main.tf

#######################################################
#### THIS IS NOT HOW YOU DEPLOY K3S IN PROD
#### THIS DOES NOT USE CERTS FOR INTERNAL COMMUNICATION
#### USE THE SECURE SCRIPT FOR ACTUAL DEPLOYMENT
####
#### By Sagnik Bhattacharya, 2024
####
#######################################################

# instaling dependency
terraform {
  required_version = ">= 0.14.0"
  required_providers {
    openstack = {
      source  = "terraform-provider-openstack/openstack"
      version = ">= 2.0.0"
    }
    tls = {
      source  = "hashicorp/tls"
      version = ">= 3.1.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.0"
    }
  }
}

provider "openstack" {
  auth_url    = var.auth_url
  region      = var.region
  tenant_name = var.tenant_name
  user_name   = var.user_name
  password    = var.password
  domain_name = var.domain_name
  insecure    = true # DANGER
}

variable "auth_url" {
  description = "OpenStack authentication URL"
  type        = string
}

variable "region" {
  description = "OpenStack region"
  type        = string
}

variable "tenant_name" {
  description = "OpenStack tenant name"
  type        = string
}

variable "user_name" {
  description = "OpenStack username"
  type        = string
}

variable "password" {
  description = "OpenStack password"
  type        = string
  sensitive   = true
}

variable "domain_name" {
  description = "OpenStack domain name"
  type        = string
}

# Broken for some reason dont know why 
# variable "ssh_public_key" {
#   description = "Path to the SSH public key"
#   type        = string
# }

variable "num_worker_nodes" {
  description = "Number of worker nodes to create"
  type        = number
}

variable "master_flavor" {
  description = "Flavor for the master node"
  type        = string
}

variable "worker_flavor" {
  description = "Flavor for the worker nodes"
  type        = string
}

variable "os_image" {
  description = "OS image to use for instances"
  type        = string
}

variable "volume_size" {
  description = "Size of the volumes to create for nodes"
  type        = number
}

variable "dns_servers" {
  description = "List of DNS servers for the instances"
  type        = list(string)
}

variable "floating_ip_pool" {
  description = "Name of the floating IP pool for the instances"
  type        = string
}

variable "delay_seconds" {
  description = "The delay in seconds before creating the worker nodes"
  default     = 120
  ## This wait time has been implimented as we were not able tp set a time till the cluster comes up
  ## we clould have also checked the file but in K3S as soon as the process in executed the file is created so we cant base it
  ## thus we habe added a timer of 120s
  ## depends on cluster load and netspeed
}

# Delay resource for master
resource "null_resource" "delay_master" {
  provisioner "local-exec" {
    command = "sleep ${var.delay_seconds}"
  }
  triggers = {
    instance_id_master = openstack_compute_instance_v2.k3s_master.id
  }
}

# Delay resource for workers
resource "null_resource" "delay_workers" {
  provisioner "local-exec" {
    command = "sleep ${var.delay_seconds}"
  }
  triggers = {
    instance_id_workers = join(",", openstack_compute_instance_v2.k3s_workers.*.id)
  }
}


# Define the network
resource "openstack_networking_network_v2" "network" {
  name           = "k3s-network"
  admin_state_up = "true"
}

# Define the subnet
resource "openstack_networking_subnet_v2" "subnet" {
  name       = "k3s-subnet"
  network_id = openstack_networking_network_v2.network.id
  cidr       = "192.168.1.0/24"
  ip_version = 4
  dns_nameservers = var.dns_servers
}

# Define the router

data "openstack_networking_network_v2" "floating_ip" {
  name = var.floating_ip_pool
}

resource "openstack_networking_router_v2" "router" {
  name             = "k3s-router"
  admin_state_up   = "true"
  external_network_id  = data.openstack_networking_network_v2.floating_ip.id
}

# Connect the router to the subnet
resource "openstack_networking_router_interface_v2" "router_interface" {
  router_id = openstack_networking_router_v2.router.id
  subnet_id = openstack_networking_subnet_v2.subnet.id
}

# Adding FIP to master ## DEPRICATED 
resource "openstack_networking_floatingip_v2" "fip" {
  pool = var.floating_ip_pool
}

resource "openstack_compute_floatingip_associate_v2" "fip_assoc" {
  floating_ip = openstack_networking_floatingip_v2.fip.address
  instance_id = openstack_compute_instance_v2.k3s_master.id
}


# Creating SSH keys
resource "tls_private_key" "ssh" {
  algorithm = "ECDSA"
  ecdsa_curve = "P256"
}

# Saving key in local
resource "local_file" "private_key" {
  content  = tls_private_key.ssh.private_key_pem
  filename = "${path.module}/id_rsa"
}

# Define the keypair for SSH
resource "openstack_compute_keypair_v2" "default" {
  name       = "k3s-key"
  # public_key = file(var.ssh_public_key)
  public_key = tls_private_key.ssh.public_key_openssh
}

# Create a new security group
resource "openstack_networking_secgroup_v2" "secgroup" {
  name        = "k3s-secgroup"
  description = "Security group for k3s"
}

# # Allow SSH traffic
# resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_ssh" {
#   direction         = "ingress"
#   ethertype         = "IPv4"
#   protocol          = "tcp"
#   port_range_min    = 22
#   port_range_max    = 22
#   remote_ip_prefix  = "0.0.0.0/0"
#   security_group_id = openstack_networking_secgroup_v2.secgroup.id
# }

########### DONT DO THIS ITS VERY BAD ########################
# Allow all inbound traffic

resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_all_inbound" {
  direction         = "ingress"
  ethertype         = "IPv4"
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.secgroup.id
}
#############################################################


# Allow all outbound traffic
resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_all_outbound" {
  direction         = "egress"
  ethertype         = "IPv4"
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.secgroup.id
}

# Define the master node
resource "openstack_compute_instance_v2" "k3s_master" {
  name          = "kube-master"
  image_name    = var.os_image
  flavor_name   = var.master_flavor
  key_pair      = openstack_compute_keypair_v2.default.name
  security_groups = ["default",openstack_networking_secgroup_v2.secgroup.name]
  network {
    uuid = openstack_networking_network_v2.network.id
  }

  # This thing does all the magic, a glorified bash script XD
  user_data = <<-EOT
              #!/bin/bash
              apt-get update
              apt-get install -y curl
              echo "Before snap"
              snap install helm --classic 
              
              # Install KubeCTL
              curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
              install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
              kubectl version --client
              echo "before K3S"
              
              # Install K3s with taint if there are worker nodes
              if [ ${var.num_worker_nodes} -gt 0 ]; then
                curl -sfL https://get.k3s.io | sh -s - --node-taint key=value:NoExecute --disable traefik --disable-agent --tls-san 127.0.0.1
              else
                # Install K3s without taint, allowing the master to schedule pods
                curl -sfL https://get.k3s.io | sh -s - --disable traefik --disable-agent --tls-san 127.0.0.1
              fi

              # Wait and save the token into a file
              while [ ! -f /var/lib/rancher/k3s/server/node-token ]; do
                sleep 5
              done
              mkdir -p /var/lib/rancher/k3s/server/
              echo $(cat /var/lib/rancher/k3s/server/node-token) > /var/lib/rancher/k3s/server/token
              chmod 777 /var/lib/rancher/k3s/server/token
              ls -ltr /var/lib/rancher/k3s/server/token

              # Mount the volume at /mnt
              mkdir /mnt/data
              mkfs.ext4 /dev/vdb
              echo '/dev/vdb /mnt/data ext4 defaults 0 0' >> /etc/fstab
              mount -a

              # Adding kubeconfig
              chmod 644 /etc/rancher/k3s/k3s.yaml
              echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> /etc/profile

              EOT

  metadata = {
    instance_role = "master"
  }
}

# Define the volume for the master node
resource "openstack_blockstorage_volume_v3" "k3s_master_volume" {
  name = "k3s-master-volume"
  size = var.volume_size
}

# Attach the volume to the master node
resource "openstack_compute_volume_attach_v2" "k3s_master_volume_attach" {
  instance_id = openstack_compute_instance_v2.k3s_master.id
  volume_id   = openstack_blockstorage_volume_v3.k3s_master_volume.id
}

resource "openstack_compute_instance_v2" "k3s_workers" {
  count         = var.num_worker_nodes
  name          = "kubeworker-${count.index}"
  image_name    = var.os_image
  flavor_name   = var.worker_flavor
  key_pair      = openstack_compute_keypair_v2.default.name
  security_groups = ["default", openstack_networking_secgroup_v2.secgroup.name]
  depends_on    = [
                    openstack_compute_instance_v2.k3s_master,
                    null_resource.delay_master
                  ]

  network {
    uuid = openstack_networking_network_v2.network.id
  }

  # This script installs necessary software and prepares the mount point
  user_data = <<-EOT
              #!/bin/bash
              echo "hello"
              apt-get update
              apt-get install -y curl
              
              # Create a mount point for the attached volume
              mkdir /mnt/data
              mkfs.ext4 /dev/vdb
              echo '/dev/vdb /mnt/data ext4 defaults 0 0' >> /etc/fstab
              mount -a

              # Save the private key
              echo '${tls_private_key.ssh.private_key_pem}' > /home/ubuntu/.ssh/id_rsa
              chmod 600 /home/ubuntu/.ssh/id_rsa
              while [ -z "$TOKEN" ]; do
                TOKEN=$(ssh -o StrictHostKeyChecking=no -i /home/ubuntu/.ssh/id_rsa ubuntu@${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4} 'sudo cat /var/lib/rancher/k3s/server/token')
                sleep 5
              done
              curl -sfL https://get.k3s.io | K3S_URL=https://${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4}:6443 K3S_TOKEN=$TOKEN sh -
              EOT

  # provisioner "remote-exec" {
  #   inline = [
  #     "TOKEN=$(ssh -o StrictHostKeyChecking=no -l ubuntu ${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4} 'cat /var/lib/rancher/k3s/server/token')",
  #     "curl -sfL https://get.k3s.io | K3S_URL=http://${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4}:6443 K3S_TOKEN=$TOKEN sh -"
  #   ]

    connection {
      type        = "ssh"
      user        = "ubuntu"
      private_key = tls_private_key.ssh.private_key_pem
      host        = self.access_ip_v4
    }

  metadata = {
    instance_role = "worker"
  }
}

# Define the volumes for the worker nodes
resource "openstack_blockstorage_volume_v3" "k3s_worker_volumes" {
  count = var.num_worker_nodes
  name  = "k3s-worker-volume-${count.index}"
  size  = var.volume_size
}

# Attach the volumes to the worker nodes
resource "openstack_compute_volume_attach_v2" "k3s_worker_volume_attach" {
  count       = var.num_worker_nodes
  instance_id = element(openstack_compute_instance_v2.k3s_workers.*.id, count.index)
  volume_id   = element(openstack_blockstorage_volume_v3.k3s_worker_volumes.*.id, count.index)
  
  # Ensure attachment only happens after instance and volume creation
  depends_on = [
    openstack_compute_instance_v2.k3s_workers,
    openstack_blockstorage_volume_v3.k3s_worker_volumes
  ]
}

## Works till here

data "kubernetes_namespace" "existing" {
  metadata {
    name = "kube-system"
  }
}

resource "kubernetes_namespace" "default" {
  count = data.kubernetes_namespace.existing.id != null ? 0 : 1
  depends_on = [null_resource.delay_workers]
  metadata {
    name = "kube-system"
  }
}


resource "kubernetes_deployment" "traefik" {
  metadata {
    name      = "traefik"
    namespace = "kube-system"
    labels = {
      app = "traefik"
    }
  }

  spec {
    replicas = 1
    selector {
      match_labels = {
        app = "traefik"
      }
    }

    template {
      metadata {
        labels = {
          app = "traefik"
        }
      }

      spec {
        container {
          name  = "traefik"
          image = "traefik:v2.4"
          args  = ["--providers.kubernetescrd", "--entrypoints.web.Address=:80", "--entrypoints.websecure.Address=:443"]

          port {
            name = "web"
            container_port = 80
          }

          port {
            name = "websecure"
            container_port = 443
          }
        }
      }
    }
  }
}

resource "kubernetes_service" "traefik" {
  metadata {
    name      = "traefik"
    namespace = "kube-system"
    labels = {
      app = "traefik"
    }
  }

  spec {
    selector = {
      app = "traefik"
    }

    type = "LoadBalancer"

    port {
      name = "web"
      port        = 80
      target_port = 80
    }

    port {
      name = "websecure"
      port        = 443
      target_port = 443
    }
  }
}

output "traefik_lb_ip" {
  value = flatten([for s in kubernetes_service.traefik.status : [for i in s.load_balancer.ingress : i.ip]])
}