diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 00000000..f225c065 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,41 @@ +# Kubernetes Notes + +## Pre + +``` +minikube start +minikube dashboard +``` + +Cleanup (optional for both): + +``` +minikube stop +minikube delete +``` + +## Non-persistent storage + +``` +kubectl apply -f k8s/redis-master-deployment.yaml +kubectl apply -f k8s/redis-master-service.yaml +kubectl apply -f k8s/redis-slave-deployment.yaml +kubectl apply -f k8s/redis-slave-service.yaml +``` + +To check (separate windows): + +``` +kubectl port-forward deployment/redis-master 7000:6379 +redis-cli -p 7000 +``` + +Clean up: + +``` +kubectl delete deployment -l app=gitenter +kubectl delete service -l app=gitenter +``` + +https://kubernetes.io/docs/tutorials/stateless-application/guestbook/ +https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/#creating-redis-deployment-and-service diff --git a/k8s/configmaps/aws-auth-cm.yaml b/k8s/configmaps/aws-auth-cm.yaml new file mode 100644 index 00000000..6054454b --- /dev/null +++ b/k8s/configmaps/aws-auth-cm.yaml @@ -0,0 +1,15 @@ +# https://docs.aws.amazon.com/eks/latest/userguide/getting-started-console.html#eks-launch-workers +# https://docs.aws.amazon.com/eks/latest/userguide/add-user-role.html +# https://docs.aws.amazon.com/eks/latest/userguide/launch-workers.html +apiVersion: v1 +kind: ConfigMap +metadata: + name: aws-auth + namespace: kube-system +data: + mapRoles: | + - rolearn: arn:aws:iam::662490392829:role/serviceRoleForEKSNodeInstanceForEC2 + username: system:node:{{EC2PrivateDNSName}} + groups: + - system:bootstrappers + - system:nodes diff --git a/k8s/nginx.yml b/k8s/nginx.yml new file mode 100644 index 00000000..d84f7de1 --- /dev/null +++ b/k8s/nginx.yml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + labels: + app: gitenter +spec: + selector: + matchLabels: + app: gitenter + replicas: 2 # tells deployment to run 2 pods matching the template + template: + metadata: + labels: + app: gitenter + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 diff --git a/k8s/redis-master-deployment.yaml b/k8s/redis-master-deployment.yaml new file mode 100644 index 00000000..510fe774 --- /dev/null +++ b/k8s/redis-master-deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis-master + labels: + app: gitenter +spec: + selector: + matchLabels: + app: gitenter + role: master + tier: backend + replicas: 1 + template: + metadata: + labels: + app: gitenter + role: master + tier: backend + spec: + containers: + - name: master + image: redis:5.0.5 + resources: + requests: + cpu: 100m + memory: 100Mi + ports: + - containerPort: 6379 diff --git a/k8s/redis-master-service.yaml b/k8s/redis-master-service.yaml new file mode 100644 index 00000000..0e935f7b --- /dev/null +++ b/k8s/redis-master-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-master + labels: + app: gitenter + role: master + tier: backend +spec: + ports: + - port: 6379 + targetPort: 6379 + selector: + app: gitenter + role: master + tier: backend diff --git a/k8s/redis-slave-deployment.yaml b/k8s/redis-slave-deployment.yaml new file mode 100644 index 00000000..f325dcc8 --- /dev/null +++ b/k8s/redis-slave-deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis-slave + labels: + app: gitenter +spec: + selector: + matchLabels: + app: gitenter + role: slave + tier: backend + replicas: 2 + template: + metadata: + labels: + app: gitenter + role: slave + tier: backend + spec: + containers: + - name: slave + image: redis:5.0.5 + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: GET_HOSTS_FROM + value: dns + # Using `GET_HOSTS_FROM=dns` requires your cluster to + # provide a dns service. As of Kubernetes 1.3, DNS is a built-in + # service launched automatically. However, if the cluster you are using + # does not have a built-in DNS service, you can instead + # access an environment variable to find the master + # service's host. To do so, comment out the 'value: dns' line above, and + # uncomment the line below: + # value: env + ports: + - containerPort: 6379 diff --git a/k8s/redis-slave-service.yaml b/k8s/redis-slave-service.yaml new file mode 100644 index 00000000..76fe73de --- /dev/null +++ b/k8s/redis-slave-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-slave + labels: + app: gitenter + role: slave + tier: backend +spec: + ports: + - port: 6379 + selector: + app: gitenter + role: slave + tier: backend diff --git a/tf-config/live/eks-staging/README.md b/tf-config/live/eks-staging/README.md new file mode 100644 index 00000000..f0e4151f --- /dev/null +++ b/tf-config/live/eks-staging/README.md @@ -0,0 +1,14 @@ +``` +cd ~/Workspace/gitenter/ +cd k8s/configmaps +kubectl apply -f aws-auth-cm.yaml +kubectl describe configmap -n kube-system aws-auth +``` + +``` +kubectl get pods --all-namespaces +kubectl -n kube-system describe pods coredns-8455f84f99-4cvct +# will error out the same as deploying a normal pod: no nodes available to schedule pods +``` + +TODO: Still "worker nodes fail to join cluster" diff --git a/tf-config/live/eks-staging/main.tf b/tf-config/live/eks-staging/main.tf new file mode 100644 index 00000000..18c8bb69 --- /dev/null +++ b/tf-config/live/eks-staging/main.tf @@ -0,0 +1,16 @@ +terraform { + required_version = "> 0.11.12" +} + +provider "aws" { + access_key = "${var.access_key}" + secret_key = "${var.secret_key}" + region = "${var.aws_region}" + version = "~> 1.35" +} + +module "eks" { + source = "../../modules/eks" + environment = "staging" + aws_region = "${var.aws_region}" +} diff --git a/tf-config/live/eks-staging/output.tf b/tf-config/live/eks-staging/output.tf new file mode 100644 index 00000000..91a1ca55 --- /dev/null +++ b/tf-config/live/eks-staging/output.tf @@ -0,0 +1,3 @@ +output "kubeconfig" { + value = "${module.eks.kubeconfig}" +} diff --git a/tf-config/live/eks-staging/variables.tf b/tf-config/live/eks-staging/variables.tf new file mode 100644 index 00000000..8ec972bc --- /dev/null +++ b/tf-config/live/eks-staging/variables.tf @@ -0,0 +1,6 @@ +variable "access_key" {} +variable "secret_key" {} +variable "aws_region" { + default = "us-east-1" + description = "AWS region e.g. us-east-1 (Please specify a region supported by the Fargate launch type)" +} diff --git a/tf-config/modules/eks/asg.tf b/tf-config/modules/eks/asg.tf new file mode 100644 index 00000000..fa6fb865 --- /dev/null +++ b/tf-config/modules/eks/asg.tf @@ -0,0 +1,85 @@ +locals { + autoscaling_group_name = "${local.main_resource_name}" + + # https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh + bootstrap_arguments = "" +} + +data "aws_ami" "eks_optimized_amis" { + owners = ["602401143452"] # Amazon EKS AMI Account ID + most_recent = true + + filter { + name = "name" + values = ["amazon-eks-node-${aws_eks_cluster.main.version}-v*"] + } +} + +# https://docs.aws.amazon.com/eks/latest/userguide/getting-started-console.html#eks-launch-workers +# https://amazon-eks.s3-us-west-2.amazonaws.com/cloudformation/2019-10-08/amazon-eks-nodegroup.yaml +resource "aws_launch_configuration" "main" { + name_prefix = "${local.main_resource_name}-" + iam_instance_profile = "${aws_iam_instance_profile.main.name}" + security_groups = ["${aws_security_group.eks_node.id}"] + + image_id = "${data.aws_ami.eks_optimized_amis.id}" + instance_type = "t2.small" + + user_data = < BlockDeviceMappings: + # > - DeviceName: /dev/xvda + # > Ebs: + # > DeleteOnTermination: true + # > VolumeSize: !Ref NodeVolumeSize + # > VolumeType: gp2 + + depends_on = [ + "aws_eks_cluster.main" + ] +} + +resource "aws_autoscaling_group" "main" { + name = "${local.autoscaling_group_name}" + launch_configuration = "${aws_launch_configuration.main.id}" + vpc_zone_identifier = ["${aws_subnet.public.*.id}"] + + min_size = 1 + max_size = 2 + desired_capacity = 2 + + tag { + key = "Name" + value = "${local.autoscaling_group_name}" + propagate_at_launch = true + } + + tag { + key = "kubernetes.io/cluster/${local.eks_cluster_name}" + value = "owned" + propagate_at_launch = true + } + + # TODO: + # > UpdatePolicy: + # > AutoScalingRollingUpdate: + # > MaxBatchSize: "1" + # > MinInstancesInService: !Ref NodeAutoScalingGroupDesiredCapacity + # > PauseTime: PT5M +} diff --git a/tf-config/modules/eks/eks.tf b/tf-config/modules/eks/eks.tf new file mode 100644 index 00000000..ee8e2d1a --- /dev/null +++ b/tf-config/modules/eks/eks.tf @@ -0,0 +1,9 @@ +resource "aws_eks_cluster" "main" { + name = "${local.eks_cluster_name}" + role_arn = "${data.aws_iam_role.eks_service.arn}" + + vpc_config { + security_group_ids = ["${aws_security_group.eks_cluster_control_panel.id}"] + subnet_ids = ["${aws_subnet.public.*.id}"] + } +} diff --git a/tf-config/modules/eks/iam.tf b/tf-config/modules/eks/iam.tf new file mode 100644 index 00000000..735538a4 --- /dev/null +++ b/tf-config/modules/eks/iam.tf @@ -0,0 +1,14 @@ +# This role is defined in live/iam-terraform-config +data "aws_iam_role" "eks_service" { + name = "serviceRoleForEKS" +} + +# This role is defined in live/iam-terraform-config +data "aws_iam_role" "eks_node_instance" { + name = "serviceRoleForEKSNodeInstanceForEC2" +} + +resource "aws_iam_instance_profile" "main" { + name = "${local.main_resource_name}" + role = "${data.aws_iam_role.eks_node_instance.name}" +} diff --git a/tf-config/modules/eks/network.tf b/tf-config/modules/eks/network.tf new file mode 100644 index 00000000..fe2426f6 --- /dev/null +++ b/tf-config/modules/eks/network.tf @@ -0,0 +1,105 @@ +# Fetch AZs in the current region +data "aws_availability_zones" "available" {} + +# Several options are provided for a user-defined VPC: +# (1) VPC with a single public subnet +# (2) VPC with public and private subnets +# (3) More complicated cases +# https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Scenarios.html +# +# To hold RDS instance in user-defined VPC we need +# at least two subnets, each in a separate availability zone. +# https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_SettingUp.html#CHAP_SettingUp.Requirements +# https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_VPC.html +# +# For chosen (2) it will use private subnet we needs to use NAT gateway +# (`aws_nat_gateway` Terraform resource), with the charging rate +# $400/year plus data. + + +# VPC in which containers will be networked. It has two public subnets. +# We distribute the subnets across the first two available subnets +# for the region, for high availability. +resource "aws_vpc" "main" { + # Private IPv4 address ranges by RFC 1918. + # Netmask can be `/16` or smaller. The largest chosen range includes: + # - 10.0.0.0/8 + # - 172.16.0.0/12 + # - 192.168.0.0/16 + # https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing + # https://en.wikipedia.org/wiki/Private_network#Private_IPv4_address_spaces + cidr_block = "10.0.0.0/16" # 65536 available addresses + + assign_generated_ipv6_cidr_block = false + enable_dns_support = true + enable_dns_hostnames = true + + # Decide whether instances launched into your VPC are run on shared or + # dedicated hardware. Choose "default" or "dedicated". Dedicated + # tenancy incurs additional costs. + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/dedicated-instance.html + instance_tenancy = "default" + + # https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html#vpc-tagging + tags = "${ + map( + "Name", "${local.main_resource_name}", + "Environment", "${var.environment}", + "kubernetes.io/cluster/${local.eks_cluster_name}", "shared", + ) + }" +} + +resource "aws_subnet" "public" { + vpc_id = "${aws_vpc.main.id}" + map_public_ip_on_launch = true + + # If two AZs: + # count.index=0: `10.0.0.0/24` - 256 addresses + # count.index=1: `10.0.1.0/24` + # https://www.terraform.io/docs/configuration/functions/cidrsubnet.html + count = "${var.az_count}" + cidr_block = "${cidrsubnet(aws_vpc.main.cidr_block, 8, count.index)}" + # Therefore, subnets will be each in a separate availability zone. + availability_zone = "${data.aws_availability_zones.available.names[count.index]}" + + # https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html#vpc-subnet-tagging + tags = "${ + map( + "Name", "${local.main_resource_name}-public", + "Environment", "${var.environment}", + "kubernetes.io/cluster/${local.eks_cluster_name}", "shared", + ) + }" +} + +# IGW for the public subnet +# +# Setup networking resources for the public subnets. Containers +# in the public subnets have public IP addresses and the routing table +# sends network traffic via the internet gateway. +resource "aws_internet_gateway" "main" { + vpc_id = "${aws_vpc.main.id}" +} + +resource "aws_route_table" "public" { + vpc_id = "${aws_vpc.main.id}" +} + +# Route the public subnet trafic through the IGW +resource "aws_route" "internet_access" { + # This is using the main route table associated with this VPC. + # This is the default one since it has not been changed by + # `aws_main_route_table_association`. + # https://www.terraform.io/docs/providers/aws/r/vpc.html#main_route_table_id + route_table_id = "${aws_route_table.public.id}" + gateway_id = "${aws_internet_gateway.main.id}" + + destination_cidr_block = "0.0.0.0/0" +} + +resource "aws_route_table_association" "private" { + count = "${var.az_count}" + subnet_id = "${element(aws_subnet.public.*.id, count.index)}" + route_table_id = "${aws_route_table.public.id}" +} diff --git a/tf-config/modules/eks/output.tf b/tf-config/modules/eks/output.tf new file mode 100644 index 00000000..0bbc1b4e --- /dev/null +++ b/tf-config/modules/eks/output.tf @@ -0,0 +1,34 @@ +locals { + kubeconfig = <` +# +# TODO: +# Should setup another jobs to add key pairs from different machines. May +# associate with login username. Also, should gather key pairs and all of them +# under a group can SSH into a particular set of machines created by +# `aws_launch_configuration`. +resource "aws_key_pair" "terraform-seashore" { + key_name = "terraform-key_pair-seashore" + public_key = "${file("~/.ssh/id_rsa.pub")}" +} diff --git a/tf-config/modules/eks/variables.tf b/tf-config/modules/eks/variables.tf new file mode 100644 index 00000000..df7a001c --- /dev/null +++ b/tf-config/modules/eks/variables.tf @@ -0,0 +1,19 @@ +variable "environment" { + description = "Prefix to distinguish different environments. E.g., `dev`, `test`, `staging`, `prod`." +} + +variable "aws_region" { + default = "us-east-1" +} + +variable "az_count" { + default = 2 +} + +locals { + name_prefix = "${var.environment}-eks" + main_resource_name = "${local.name_prefix}" + + # Below variables are cross-used in different Terraform file. + eks_cluster_name = "${local.main_resource_name}" +} diff --git a/tf-config/modules/iam-terraform-config-group/stateless.tf b/tf-config/modules/iam-terraform-config-group/stateless.tf index 4b939ca9..8c457792 100644 --- a/tf-config/modules/iam-terraform-config-group/stateless.tf +++ b/tf-config/modules/iam-terraform-config-group/stateless.tf @@ -74,6 +74,33 @@ resource "aws_iam_group_policy_attachment" "ecs_instance_attach" { policy_arn = "${data.aws_iam_policy.ecs_instance.arn}" } +resource "aws_iam_policy" "eks" { + name = "AmazonEKS_FullAccess" + path = "/" + + policy = <