added in some services and updates

warehouse/terraform/main.tf

@@ -1,6 +1,6 @@
 // S3 Bucket
-resource "aws_s3_bucket" "my_bucket" {
+resource "aws_s3_bucket" "warehouse_bucket" {
-  bucket = "my-unique-bucket-name"
+  bucket = "buster-warehouse"
 }
 
 // VPC and Subnets
@@ -8,7 +8,7 @@ module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
   version = "3.14.0"
 
-  name = "my-vpc"
+  name = "buster-warehouse-vpc"
   cidr = var.vpc_cidr
 
   azs             = ["${var.region}a", "${var.region}b", "${var.region}c"]
@@ -91,12 +91,13 @@ module "vpc" {
   tags = {
     Terraform   = "true"
     Environment = "dev"
+    Project     = "buster-warehouse"
   }
 }
 
 // Security Group for Load Balancer
 resource "aws_security_group" "lb_sg" {
-  name_prefix = "eks-lb-sg"
+  name_prefix = "buster-warehouse-lb-sg"
   vpc_id      = module.vpc.vpc_id
 
   ingress {
@@ -124,13 +125,14 @@ resource "aws_security_group" "lb_sg" {
   }
 
   tags = {
-    Name = "eks-lb-sg"
+    Name    = "buster-warehouse-lb-sg"
+    Project = "buster-warehouse"
   }
 }
 
 // Security Group for Frontend Nodes
 resource "aws_security_group" "fe_sg" {
-  name_prefix = "eks-fe-sg"
+  name_prefix = "buster-warehouse-fe-sg"
   vpc_id      = module.vpc.vpc_id
 
   ingress {
@@ -150,13 +152,14 @@ resource "aws_security_group" "fe_sg" {
   }
 
   tags = {
-    Name = "eks-fe-sg"
+    Name    = "buster-warehouse-fe-sg"
+    Project = "buster-warehouse"
   }
 }
 
 // Security Group for Backend Nodes
 resource "aws_security_group" "be_sg" {
-  name_prefix = "eks-be-sg"
+  name_prefix = "buster-warehouse-be-sg"
   vpc_id      = module.vpc.vpc_id
 
   // Add rules as needed for backend communication
@@ -170,7 +173,8 @@ resource "aws_security_group" "be_sg" {
   }
 
   tags = {
-    Name = "eks-be-sg"
+    Name    = "buster-warehouse-be-sg"
+    Project = "buster-warehouse"
   }
 }
 
@@ -179,7 +183,7 @@ module "eks" {
   source  = "terraform-aws-modules/eks/aws"
   version = "18.26.3"
 
-  cluster_name    = var.cluster_name
+  cluster_name    = "buster-warehouse-${var.cluster_name}"
   cluster_version = "1.22"
 
   vpc_id     = module.vpc.vpc_id
@@ -201,10 +205,12 @@ module "eks" {
       labels = {
         NodeGroup = key
         NodeType  = value.instance_type
+        Project   = "buster-warehouse"
       }
 
       tags = {
         NodeGroup = key
+        Project   = "buster-warehouse"
       }
 
       vpc_security_group_ids = [
@@ -218,14 +224,15 @@ module "eks" {
 
 // Application Load Balancer
 resource "aws_lb" "eks_alb" {
-  name               = "eks-alb"
+  name               = "buster-warehouse-alb"
   internal           = false
   load_balancer_type = "application"
   security_groups    = [aws_security_group.lb_sg.id]
   subnets            = module.vpc.public_subnets
 
   tags = {
-    Name = "eks-alb"
+    Name    = "buster-warehouse-alb"
+    Project = "buster-warehouse"
   }
 }
 
@@ -241,7 +248,7 @@ resource "aws_lb_listener" "front_end" {
 }
 
 resource "aws_lb_target_group" "fe_tg" {
-  name     = "fe-tg"
+  name     = "buster-warehouse-fe-tg"
   port     = 9030
   protocol = "HTTP"
   vpc_id   = module.vpc.vpc_id
@@ -263,7 +270,7 @@ resource "aws_autoscaling_attachment" "fe_asg_attachment" {
 // Kubernetes Deployment for Docker image
 resource "kubernetes_deployment" "example" {
   metadata {
-    name = "example-deployment"
+    name = "buster-warehouse-deployment"
   }
 
   spec {
@@ -301,13 +308,14 @@ resource "aws_vpc_endpoint" "s3" {
   route_table_ids   = module.vpc.private_route_table_ids
 
   tags = {
-    Name = "s3-endpoint"
+    Name    = "buster-warehouse-s3-endpoint"
+    Project = "buster-warehouse"
   }
 }
 
 // Update S3 bucket policy to allow access from the VPC Endpoint
 resource "aws_s3_bucket_policy" "allow_access_from_vpc" {
-  bucket = aws_s3_bucket.my_bucket.id
+  bucket = aws_s3_bucket.warehouse_bucket.id
 
   policy = jsonencode({
     Version = "2012-10-17"
@@ -318,8 +326,8 @@ resource "aws_s3_bucket_policy" "allow_access_from_vpc" {
         Principal = "*"
         Action    = "s3:*"
         Resource = [
-          aws_s3_bucket.my_bucket.arn,
+          aws_s3_bucket.warehouse_bucket.arn,
-          "${aws_s3_bucket.my_bucket.arn}/*",
+          "${aws_s3_bucket.warehouse_bucket.arn}/*",
         ]
         Condition = {
           StringEquals = {
@@ -333,7 +341,7 @@ resource "aws_s3_bucket_policy" "allow_access_from_vpc" {
 
 // Helm Release for StarRocks
 resource "helm_release" "starrocks" {
-  name       = "starrocks"
+  name       = "buster-warehouse-starrocks"
   repository = "https://starrocks.github.io/starrocks-kubernetes-operator"
   chart      = "starrocks-operator"
   namespace  = kubernetes_namespace.starrocks.metadata[0].name
@@ -348,8 +356,129 @@ resource "helm_release" "starrocks" {
 // Create a namespace for StarRocks
 resource "kubernetes_namespace" "starrocks" {
   metadata {
-    name = "starrocks"
+    name = "buster-warehouse-starrocks"
   }
 
   depends_on = [module.eks]
 }
+
+// Helm Release for PostgreSQL
+resource "helm_release" "postgresql" {
+  name       = "buster-warehouse-postgresql"
+  repository = "https://charts.bitnami.com/bitnami"
+  chart      = "postgresql"
+  namespace  = kubernetes_namespace.postgresql.metadata[0].name
+
+  set {
+    name  = "global.postgresql.auth.postgresPassword"
+    value = var.postgres_password
+  }
+
+  set {
+    name  = "primary.persistence.size"
+    value = "10Gi"
+  }
+
+  depends_on = [module.eks, kubernetes_namespace.postgresql]
+}
+
+// Create a namespace for PostgreSQL
+resource "kubernetes_namespace" "postgresql" {
+  metadata {
+    name = "buster-warehouse-postgresql"
+  }
+
+  depends_on = [module.eks]
+}
+
+// Helm Release for Iceberg REST
+resource "helm_release" "iceberg_rest" {
+  name       = "buster-warehouse-iceberg-rest"
+  repository = "https://charts.bitnami.com/bitnami"
+  chart      = "common"
+  namespace  = kubernetes_namespace.iceberg_rest.metadata[0].name
+
+  values = [
+    <<-EOT
+    replicaCount: 1
+    image:
+      repository: tabulario/iceberg-rest
+      tag: latest
+      pullPolicy: IfNotPresent
+    service:
+      type: ClusterIP
+      port: 8181
+    containerPort: 8181
+    env:
+      - name: AWS_ACCESS_KEY_ID
+        valueFrom:
+          secretKeyRef:
+            name: aws-credentials
+            key: aws-access-key-id
+      - name: AWS_SECRET_ACCESS_KEY
+        valueFrom:
+          secretKeyRef:
+            name: aws-credentials
+            key: aws-secret-access-key
+      - name: AWS_REGION
+        value: "${var.region}"
+      - name: CATALOG_WAREHOUSE
+        value: "${aws_s3_bucket.warehouse_bucket.id}"
+      - name: CATALOG_URI
+        value: "jdbc:postgresql://${helm_release.postgresql.name}-postgresql.${kubernetes_namespace.postgresql.metadata[0].name}.svc.cluster.local:5432/postgres"
+      - name: CATALOG_JDBC_USER
+        value: "postgres"
+      - name: CATALOG_JDBC_PASSWORD
+        value: "${var.postgres_password}"
+    EOT
+  ]
+
+  depends_on = [module.eks, kubernetes_namespace.iceberg_rest, helm_release.postgresql]
+}
+
+// Create a namespace for Iceberg REST
+resource "kubernetes_namespace" "iceberg_rest" {
+  metadata {
+    name = "buster-warehouse-iceberg-rest"
+  }
+
+  depends_on = [module.eks]
+}
+
+// ... existing code ...
+
+// Helm Release for Nginx Ingress Controller
+resource "helm_release" "nginx_ingress" {
+  name       = "buster-warehouse-nginx-ingress"
+  repository = "https://kubernetes.github.io/ingress-nginx"
+  chart      = "ingress-nginx"
+  namespace  = kubernetes_namespace.nginx_ingress.metadata[0].name
+
+  set {
+    name  = "controller.service.type"
+    value = "LoadBalancer"
+  }
+
+  set {
+    name  = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-type"
+    value = "nlb"
+  }
+
+  set {
+    name  = "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-cross-zone-load-balancing-enabled"
+    value = "true"
+  }
+
+  depends_on = [module.eks, kubernetes_namespace.nginx_ingress]
+}
+
+// Create a namespace for Nginx Ingress
+resource "kubernetes_namespace" "nginx_ingress" {
+  metadata {
+    name = "buster-warehouse-nginx-ingress"
+  }
+
+  depends_on = [module.eks]
+}
+
+// ... rest of the existing code ...

warehouse/terraform/variables.tf

@@ -3,7 +3,7 @@ variable "region" {
 }
 
 variable "cluster_name" {
-  default = "my-eks-cluster"
+  default = "buster-warehouse"
 }
 
 variable "vpc_cidr" {
@@ -27,27 +27,47 @@ variable "node_groups" {
   }))
   default = {
     fe_group = {
-      instance_type = "r6g.2xlarge"
+      instance_type = "t3.small"
       min_size      = 1
-      max_size      = 3
+      max_size      = 1
-      desired_size  = 2
+      desired_size  = 1
       disk_size     = 200
     },
     be_group = {
-      instance_type = "r6g.4xlarge"
+      instance_type = "t3.small"
       min_size      = 1
-      max_size      = 3
+      max_size      = 1
-      desired_size  = 2
+      desired_size  = 1
       disk_size     = 1000
     },
     lb_group = {
       instance_type = "t3.small"
       min_size      = 1
-      max_size      = 2
+      max_size      = 1
+      desired_size  = 1
+      disk_size     = 20
+    }
+    postgresql_group = {
+      instance_type = "t3.small"
+      min_size      = 1
+      max_size      = 1
+      desired_size  = 1
+      disk_size     = 20
+    }
+    iceberg_rest_group = {
+      instance_type = "t3.small"
+      min_size      = 1
+      max_size      = 1
       desired_size  = 1
       disk_size     = 20
     }
   }
 }
 
+variable "postgres_password" {
+  description = "password"
+  type        = string
+  sensitive   = true
+}
+
 // Add more variables as needed