From b84f53b07e782c3d53ab187a4ad93c6b503f8cc7 Mon Sep 17 00:00:00 2001 From: Rick Date: Wed, 31 Jul 2024 08:37:21 +0200 Subject: [PATCH] -- added k3s secure and insecure as of now use insecure --- terraform/K3S/insecure.main.tf | 483 ++++++++++++++++++++++++++++++++ terraform/K3S/runner/nginx.yaml | 111 ++++++++ terraform/K3S/secure.main.tf | 112 ++++++++ terraform/K3S/terraform.tfvars | 29 ++ 4 files changed, 735 insertions(+) create mode 100644 terraform/K3S/insecure.main.tf create mode 100644 terraform/K3S/runner/nginx.yaml create mode 100644 terraform/K3S/secure.main.tf create mode 100644 terraform/K3S/terraform.tfvars diff --git a/terraform/K3S/insecure.main.tf b/terraform/K3S/insecure.main.tf new file mode 100644 index 0000000..f30f31f --- /dev/null +++ b/terraform/K3S/insecure.main.tf @@ -0,0 +1,483 @@ +####################################################### +#### THIS IS NOT HOW YOU DEPLOY K3S IN PROD +#### THIS DOES NOT USE CERTS FOR INTERNAL COMMUNICATION +#### USE THE SECURE SCRIPT FOR ACTUAL DEPLOYMENT +#### +#### By Sagnik Bhattacharya, 2024 +#### +####################################################### + +# instaling dependency +terraform { + required_version = ">= 0.14.0" + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = ">= 2.0.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 3.1.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + } +} + +provider "openstack" { + auth_url = var.auth_url + region = var.region + tenant_name = var.tenant_name + user_name = var.user_name + password = var.password + domain_name = var.domain_name + insecure = true # DANGER +} + +variable "auth_url" { + description = "OpenStack authentication URL" + type = string +} + +variable "region" { + description = "OpenStack region" + type = string +} + +variable "tenant_name" { + description = "OpenStack tenant name" + type = string +} + +variable "user_name" { + description = "OpenStack username" + type = string +} + +variable "password" { + description = "OpenStack password" + type = string + sensitive = true +} + +variable "domain_name" { + description = "OpenStack domain name" + type = string +} + +# Broken for some reason dont know why +# variable "ssh_public_key" { +# description = "Path to the SSH public key" +# type = string +# } + +variable "num_worker_nodes" { + description = "Number of worker nodes to create" + type = number +} + +variable "master_flavor" { + description = "Flavor for the master node" + type = string +} + +variable "worker_flavor" { + description = "Flavor for the worker nodes" + type = string +} + +variable "os_image" { + description = "OS image to use for instances" + type = string +} + +variable "volume_size" { + description = "Size of the volumes to create for nodes" + type = number +} + +variable "dns_servers" { + description = "List of DNS servers for the instances" + type = list(string) +} + +variable "floating_ip_pool" { + description = "Name of the floating IP pool for the instances" + type = string +} + +variable "delay_seconds" { + description = "The delay in seconds before creating the worker nodes" + default = 120 + ## This wait time has been implimented as we were not able tp set a time till the cluster comes up + ## we clould have also checked the file but in K3S as soon as the process in executed the file is created so we cant base it + ## thus we habe added a timer of 120s + ## depends on cluster load and netspeed +} + +# Delay resource for master +resource "null_resource" "delay_master" { + provisioner "local-exec" { + command = "sleep ${var.delay_seconds}" + } + triggers = { + instance_id_master = openstack_compute_instance_v2.k3s_master.id + } +} + +# Delay resource for workers +resource "null_resource" "delay_workers" { + provisioner "local-exec" { + command = "sleep ${var.delay_seconds}" + } + triggers = { + instance_id_workers = join(",", openstack_compute_instance_v2.k3s_workers.*.id) + } +} + + +# Define the network +resource "openstack_networking_network_v2" "network" { + name = "k3s-network" + admin_state_up = "true" +} + +# Define the subnet +resource "openstack_networking_subnet_v2" "subnet" { + name = "k3s-subnet" + network_id = openstack_networking_network_v2.network.id + cidr = "192.168.1.0/24" + ip_version = 4 + dns_nameservers = var.dns_servers +} + +# Define the router + +data "openstack_networking_network_v2" "floating_ip" { + name = var.floating_ip_pool +} + +resource "openstack_networking_router_v2" "router" { + name = "k3s-router" + admin_state_up = "true" + external_network_id = data.openstack_networking_network_v2.floating_ip.id +} + +# Connect the router to the subnet +resource "openstack_networking_router_interface_v2" "router_interface" { + router_id = openstack_networking_router_v2.router.id + subnet_id = openstack_networking_subnet_v2.subnet.id +} + +# Adding FIP to master ## DEPRICATED +resource "openstack_networking_floatingip_v2" "fip" { + pool = var.floating_ip_pool +} + +resource "openstack_compute_floatingip_associate_v2" "fip_assoc" { + floating_ip = openstack_networking_floatingip_v2.fip.address + instance_id = openstack_compute_instance_v2.k3s_master.id +} + + +# Creating SSH keys +resource "tls_private_key" "ssh" { + algorithm = "ECDSA" + ecdsa_curve = "P256" +} + +# Saving key in local +resource "local_file" "private_key" { + content = tls_private_key.ssh.private_key_pem + filename = "${path.module}/id_rsa" +} + +# Define the keypair for SSH +resource "openstack_compute_keypair_v2" "default" { + name = "k3s-key" + # public_key = file(var.ssh_public_key) + public_key = tls_private_key.ssh.public_key_openssh +} + +# Create a new security group +resource "openstack_networking_secgroup_v2" "secgroup" { + name = "k3s-secgroup" + description = "Security group for k3s" +} + +# # Allow SSH traffic +# resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_ssh" { +# direction = "ingress" +# ethertype = "IPv4" +# protocol = "tcp" +# port_range_min = 22 +# port_range_max = 22 +# remote_ip_prefix = "0.0.0.0/0" +# security_group_id = openstack_networking_secgroup_v2.secgroup.id +# } + +########### DONT DO THIS ITS VERY BAD ######################## +# Allow all inbound traffic + +resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_all_inbound" { + direction = "ingress" + ethertype = "IPv4" + remote_ip_prefix = "0.0.0.0/0" + security_group_id = openstack_networking_secgroup_v2.secgroup.id +} +############################################################# + + +# Allow all outbound traffic +resource "openstack_networking_secgroup_rule_v2" "secgroup_rule_all_outbound" { + direction = "egress" + ethertype = "IPv4" + remote_ip_prefix = "0.0.0.0/0" + security_group_id = openstack_networking_secgroup_v2.secgroup.id +} + +# Define the master node +resource "openstack_compute_instance_v2" "k3s_master" { + name = "kube-master" + image_name = var.os_image + flavor_name = var.master_flavor + key_pair = openstack_compute_keypair_v2.default.name + security_groups = ["default",openstack_networking_secgroup_v2.secgroup.name] + network { + uuid = openstack_networking_network_v2.network.id + } + + # This thing does all the magic, a glorified bash script XD + user_data = <<-EOT + #!/bin/bash + apt-get update + apt-get install -y curl + echo "Before snap" + snap install helm --classic + + # Install KubeCTL + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl version --client + echo "before K3S" + + # Install K3s with taint if there are worker nodes + if [ ${var.num_worker_nodes} -gt 0 ]; then + curl -sfL https://get.k3s.io | sh -s - --node-taint key=value:NoExecute --disable traefik --disable-agent --tls-san 127.0.0.1 + else + # Install K3s without taint, allowing the master to schedule pods + curl -sfL https://get.k3s.io | sh -s - --disable traefik --disable-agent --tls-san 127.0.0.1 + fi + + # Wait and save the token into a file + while [ ! -f /var/lib/rancher/k3s/server/node-token ]; do + sleep 5 + done + mkdir -p /var/lib/rancher/k3s/server/ + echo $(cat /var/lib/rancher/k3s/server/node-token) > /var/lib/rancher/k3s/server/token + chmod 777 /var/lib/rancher/k3s/server/token + ls -ltr /var/lib/rancher/k3s/server/token + + # Mount the volume at /mnt + mkdir /mnt/data + mkfs.ext4 /dev/vdb + echo '/dev/vdb /mnt/data ext4 defaults 0 0' >> /etc/fstab + mount -a + + # Adding kubeconfig + chmod 644 /etc/rancher/k3s/k3s.yaml + echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> /etc/profile + + EOT + + metadata = { + instance_role = "master" + } +} + +# Define the volume for the master node +resource "openstack_blockstorage_volume_v3" "k3s_master_volume" { + name = "k3s-master-volume" + size = var.volume_size +} + +# Attach the volume to the master node +resource "openstack_compute_volume_attach_v2" "k3s_master_volume_attach" { + instance_id = openstack_compute_instance_v2.k3s_master.id + volume_id = openstack_blockstorage_volume_v3.k3s_master_volume.id +} + +resource "openstack_compute_instance_v2" "k3s_workers" { + count = var.num_worker_nodes + name = "kubeworker-${count.index}" + image_name = var.os_image + flavor_name = var.worker_flavor + key_pair = openstack_compute_keypair_v2.default.name + security_groups = ["default", openstack_networking_secgroup_v2.secgroup.name] + depends_on = [ + openstack_compute_instance_v2.k3s_master, + null_resource.delay_master + ] + + network { + uuid = openstack_networking_network_v2.network.id + } + + # This script installs necessary software and prepares the mount point + user_data = <<-EOT + #!/bin/bash + echo "hello" + apt-get update + apt-get install -y curl + + # Create a mount point for the attached volume + mkdir /mnt/data + mkfs.ext4 /dev/vdb + echo '/dev/vdb /mnt/data ext4 defaults 0 0' >> /etc/fstab + mount -a + + # Save the private key + echo '${tls_private_key.ssh.private_key_pem}' > /home/ubuntu/.ssh/id_rsa + chmod 600 /home/ubuntu/.ssh/id_rsa + while [ -z "$TOKEN" ]; do + TOKEN=$(ssh -o StrictHostKeyChecking=no -i /home/ubuntu/.ssh/id_rsa ubuntu@${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4} 'sudo cat /var/lib/rancher/k3s/server/token') + sleep 5 + done + curl -sfL https://get.k3s.io | K3S_URL=https://${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4}:6443 K3S_TOKEN=$TOKEN sh - + EOT + + # provisioner "remote-exec" { + # inline = [ + # "TOKEN=$(ssh -o StrictHostKeyChecking=no -l ubuntu ${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4} 'cat /var/lib/rancher/k3s/server/token')", + # "curl -sfL https://get.k3s.io | K3S_URL=http://${openstack_compute_instance_v2.k3s_master.network.0.fixed_ip_v4}:6443 K3S_TOKEN=$TOKEN sh -" + # ] + + connection { + type = "ssh" + user = "ubuntu" + private_key = tls_private_key.ssh.private_key_pem + host = self.access_ip_v4 + } + + metadata = { + instance_role = "worker" + } +} + +# Define the volumes for the worker nodes +resource "openstack_blockstorage_volume_v3" "k3s_worker_volumes" { + count = var.num_worker_nodes + name = "k3s-worker-volume-${count.index}" + size = var.volume_size +} + +# Attach the volumes to the worker nodes +resource "openstack_compute_volume_attach_v2" "k3s_worker_volume_attach" { + count = var.num_worker_nodes + instance_id = element(openstack_compute_instance_v2.k3s_workers.*.id, count.index) + volume_id = element(openstack_blockstorage_volume_v3.k3s_worker_volumes.*.id, count.index) + + # Ensure attachment only happens after instance and volume creation + depends_on = [ + openstack_compute_instance_v2.k3s_workers, + openstack_blockstorage_volume_v3.k3s_worker_volumes + ] +} + +## Works till here + +data "kubernetes_namespace" "existing" { + metadata { + name = "kube-system" + } +} + +resource "kubernetes_namespace" "default" { + count = data.kubernetes_namespace.existing.id != null ? 0 : 1 + depends_on = [null_resource.delay_workers] + metadata { + name = "kube-system" + } +} + + +resource "kubernetes_deployment" "traefik" { + metadata { + name = "traefik" + namespace = "kube-system" + labels = { + app = "traefik" + } + } + + spec { + replicas = 1 + selector { + match_labels = { + app = "traefik" + } + } + + template { + metadata { + labels = { + app = "traefik" + } + } + + spec { + container { + name = "traefik" + image = "traefik:v2.4" + args = ["--providers.kubernetescrd", "--entrypoints.web.Address=:80", "--entrypoints.websecure.Address=:443"] + + port { + name = "web" + container_port = 80 + } + + port { + name = "websecure" + container_port = 443 + } + } + } + } + } +} + +resource "kubernetes_service" "traefik" { + metadata { + name = "traefik" + namespace = "kube-system" + labels = { + app = "traefik" + } + } + + spec { + selector = { + app = "traefik" + } + + type = "LoadBalancer" + + port { + name = "web" + port = 80 + target_port = 80 + } + + port { + name = "websecure" + port = 443 + target_port = 443 + } + } +} + +output "traefik_lb_ip" { + value = flatten([for s in kubernetes_service.traefik.status : [for i in s.load_balancer.ingress : i.ip]]) +} diff --git a/terraform/K3S/runner/nginx.yaml b/terraform/K3S/runner/nginx.yaml new file mode 100644 index 0000000..00a3a58 --- /dev/null +++ b/terraform/K3S/runner/nginx.yaml @@ -0,0 +1,111 @@ +# Namespace +apiVersion: v1 +kind: Namespace +metadata: + name: nginx-deployment + +--- +# Persistent Volume +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nginx-pv +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + hostPath: + path: "/mnt/data/nginx-pv" # Adjust this path according to the Kubernetes node filesystem + +--- +# Persistent Volume Claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nginx-pvc + namespace: nginx-deployment +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + +--- +# Deployment for Nginx Pods +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + namespace: nginx-deployment +spec: + replicas: 2 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /usr/share/nginx/html + name: nginx-storage + resources: + limits: + memory: "512Mi" + cpu: "1" + requests: + memory: "256Mi" + cpu: "0.5" + volumes: + - name: nginx-storage + persistentVolumeClaim: + claimName: nginx-pvc + +--- +# Service of type LoadBalancer to expose Nginx externally +apiVersion: v1 +kind: Service +metadata: + name: nginx-service + namespace: nginx-deployment +spec: + selector: + app: nginx + ports: + - protocol: TCP + port: 80 + targetPort: 80 + type: LoadBalancer + +--- +# Ingress to route traffic to Nginx service (no specific host) +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx-ingress + namespace: nginx-deployment + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + rules: + - http: +# host: something.hs-fulda.de # Replace with your domain + paths: + - path: / + pathType: Prefix + backend: + service: + name: nginx-service + port: + number: 80 diff --git a/terraform/K3S/secure.main.tf b/terraform/K3S/secure.main.tf new file mode 100644 index 0000000..d66993b --- /dev/null +++ b/terraform/K3S/secure.main.tf @@ -0,0 +1,112 @@ +####################################################### +#### Incomplete +#### +#### By Sagnik Bhattacharya, 2024 +#### +####################################################### + +terraform { + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = "~> 1.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.0" + } + } +} + +provider "openstack" { + auth_url = var.auth_url + region = var.region + tenant_name = var.tenant_name + user_name = var.user_name + password = var.password + domain_name = var.domain_name +} + +provider "kubernetes" { + host = var.kubernetes_host + client_certificate = file(var.client_certificate) + client_key = file(var.client_key) + cluster_ca_certificate = file(var.cluster_ca_certificate) +} + +# Define variables without default values +variable "auth_url" { + description = "OpenStack authentication URL" + type = string +} + +variable "region" { + description = "OpenStack region" + type = string +} + +variable "tenant_name" { + description = "OpenStack tenant name" + type = string +} + +variable "user_name" { + description = "OpenStack username" + type = string +} + +variable "password" { + description = "OpenStack password" + type = string + sensitive = true +} + +variable "domain_name" { + description = "OpenStack domain name" + type = string +} + +variable "ssh_public_key" { + description = "Path to the SSH public key" + type = string +} + +variable "kubernetes_host" { + description = "Kubernetes API server URL" + type = string +} + +variable "client_certificate" { + description = "Path to the client certificate for Kubernetes" + type = string +} + +variable "client_key" { + description = "Path to the client key for Kubernetes" + type = string +} + +variable "cluster_ca_certificate" { + description = "Path to the cluster CA certificate for Kubernetes" + type = string +} + +variable "num_worker_nodes" { + description = "Number of worker nodes to create" + type = number +} + +variable "master_flavor" { + description = "Flavor for the master node" + type = string +} + +variable "worker_flavor" { + description = "Flavor for the worker nodes" + type = string +} + +variable "os_image" { + description = "OS image to use for instances" + type = string +} diff --git a/terraform/K3S/terraform.tfvars b/terraform/K3S/terraform.tfvars new file mode 100644 index 0000000..1580a14 --- /dev/null +++ b/terraform/K3S/terraform.tfvars @@ -0,0 +1,29 @@ +## These are for connecting with Openstack and sharing the Keypair + +auth_url = "https://10.32.4.182:5000/v3" +region = "RegionOne" +tenant_name = "CloudComp10" # Also known as project +user_name = "CloudComp10" +password = "demo" +domain_name = "default" +# ssh_public_key = "~/.ssh/id_ecdsa.pub" + +# These are needed for the internal SSL certificate +# Must use for Pord env but for simplicity removed from here + +# client_certificate = "~/.ssh/client.crt" +# client_key = "~/.ssh/client.key" +# cluster_ca_certificate = "~/.ssh/ca.crt" + +# Instance Configuration +# num_worker_nodes is < 0 then master will be the worker otherwise master +# is only for control + +num_worker_nodes = 3 +master_flavor = "m1.small" +worker_flavor = "m1.medium" +os_image = "ubuntu-22.04-jammy-x86_64" +volume_size = 15 +dns_servers = ["10.33.16.100"] +floating_ip_pool = "ext_net" +delay_seconds = 120 \ No newline at end of file