Intitial state

athena.tf

@@ -0,0 +1,27 @@
+resource "aws_s3_bucket" "athena_results" {
+    bucket = "athena-results-${var.account_number}-${var.student_initials}-${var.student_index_no}"
+        force_destroy = true
+        tags = merge(local.common_tags)
+    }
+
+resource "aws_s3_bucket_lifecycle_configuration" "athena_results_lifecycle" {
+    bucket = aws_s3_bucket.athena_results.id
+        rule {
+            id = "standard-expiration"
+            status = "Enabled"
+            expiration {
+                days=1
+            }
+        }
+    }
+
+resource "aws_athena_workgroup" "athena_workgroup" {
+    name = "development"
+    configuration {
+        enforce_workgroup_configuration = true
+        result_configuration {
+            output_location = "s3://${aws_s3_bucket.athena_results.bucket}/output/"
+        }
+    }
+    force_destroy = true
+}

glue_catalog_database.tf

@@ -0,0 +1,17 @@
+resource "aws_glue_catalog_database" "datalake_db_raw_zone" {
+name = "datalake_raw_${var.account_number}_${var.student_initials}_${var.student_index_no}"
+}
+
+resource "aws_glue_catalog_database" "datalake_db_processed_zone" {
+    name = "datalake_processed_${var.account_number}_${var.student_initials}_${var.student_index_no}"
+}
+resource "aws_glue_crawler" "glue_crawler_raw_zone" {
+    database_name = aws_glue_catalog_database.datalake_db_raw_zone.name
+    name = "gc-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
+    role = var.lab_role_arn
+    table_prefix = "crawler_"
+    s3_target {
+    path = "s3://${aws_s3_bucket.raw_bucket.bucket}/raw-zone/stockdata/"
+    }
+    tags = merge(local.common_tags, )
+}

kinesis_ds.tf

@@ -0,0 +1,16 @@
+resource "aws_kinesis_stream" "cryptostock_stream" {
+    name = "cryptostock-${var.account_number}-${var.student_initials}-${var.student_index_no}"
+    shard_count = 1
+    enforce_consumer_deletion = true
+    shard_level_metrics = [
+        "IncomingBytes",
+        "OutgoingBytes",
+        "IncomingRecords",
+        "OutgoingRecords"
+        ]
+    tags = {
+        Purpose = "UAM Cloud Data Processing"
+        Environment = "DEV"
+        Owner = var.student_full_name
+    }
+}

kinesis_fh.tf

@@ -0,0 +1,16 @@
+resource "aws_kinesis_firehose_delivery_stream" "stock_delivery_stream" {
+    name = "firehose-${var.account_number}-${var.student_initials}-${var.student_index_no}"
+    destination = "extended_s3"
+    kinesis_source_configuration {
+        kinesis_stream_arn = aws_kinesis_stream.cryptostock_stream.arn
+        role_arn = var.lab_role_arn
+    }   
+    extended_s3_configuration {
+        role_arn = var.lab_role_arn
+        bucket_arn = aws_s3_bucket.raw_bucket.arn
+        buffering_size = 1
+        buffering_interval = 60
+        prefix = "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
+        error_output_prefix = "${"raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}"}${"/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"}/"
+    }
+}

lambda.tf

@@ -0,0 +1,36 @@
+resource "aws_lambda_layer_version" "aws_wrangler" {
+    filename = "../lambda/awswrangler-layer-2.7.0-py3.8.zip"
+    layer_name = "aws_wrangler_${var.account_number}_${var.student_initials}_${var.student_index_no}"
+    source_code_hash = "${filebase64sha256("../lambda/awswrangler-layer-2.7.0-py3.8.zip")}"
+    compatible_runtimes = ["python3.8"]
+}
+
+resource "aws_lambda_function" "etl_post_processing" {
+    function_name = "etl-post-processing-${var.account_number}-${var.student_initials}-${var.student_index_no}"
+    filename = "lambda_definition.1.zip"
+    handler = "lambda_definition.etl_function"
+    runtime = "python3.8"
+    role = var.lab_role_arn
+    timeout = 300
+    memory_size = 512
+    source_code_hash= filebase64sha256("lambda_definition.1.zip")
+    layers = ["${aws_lambda_layer_version.aws_wrangler.arn}"]
+}
+
+resource "aws_lambda_permission" "allow_bucket" {
+    statement_id = "AllowExecutionFromS3Bucket"
+    action = "lambda:InvokeFunction"
+    function_name = aws_lambda_function.etl_post_processing.arn
+    principal = "s3.amazonaws.com"
+    source_arn = aws_s3_bucket.raw_bucket.arn
+}
+
+resource "aws_s3_bucket_notification" "trigger_etl_lambda" {
+    bucket = aws_s3_bucket.raw_bucket.id
+    lambda_function {
+        lambda_function_arn = aws_lambda_function.etl_post_processing.arn
+        events = ["s3:ObjectCreated:*"]
+        filter_prefix = "raw-zone/"
+    }
+    depends_on = [aws_lambda_permission.allow_bucket]
+}

main.tf

@@ -0,0 +1,7 @@
+locals {
+  common_tags = {
+    purpose = "UAM Cloud Data Processing"
+    environment = "DEV"
+    owner = var.student_full_name
+  }
+}

provider.tf

@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}
+
+provider "aws" {
+  profile = "default"
+  region = var.region
+}

s3.tf

@@ -0,0 +1,16 @@
+resource "aws_s3_bucket" "raw_bucket" {
+	bucket = "datalake-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
+	force_destroy = true
+	tags = 	{
+		Purpose = "UAM Cloud Data Processing"
+		Environment = "DEV"
+	}
+}
+resource "aws_s3_bucket" "processed_bucket" {
+	bucket = "datalake-processed-${var.account_number}-${var.student_initials}-${var.student_index_no}"
+	force_destroy = true
+	tags = 	{
+		Purpose = "UAM Cloud Data Processing"
+		Environment = "DEV"
+	}
+}

variables.tf

@@ -0,0 +1,37 @@
+variable "account_number" {
+  description = "Account number"
+  type = number
+}
+
+variable "region" {
+  description = "Region name - must be NVirginia us-east-1"
+  type = string
+  default = "us-east-1"
+}
+
+variable "environment" {
+  description = "Environment name"
+  type = string
+  default = "dev"
+}
+
+variable "student_initials" {
+  description = "letters of first and last names"
+  type = string
+}
+
+variable "student_full_name" {
+  description = "Student's full name"
+  type = string
+}
+
+variable "student_index_no" {
+  description = "Index no"
+  type = string
+}
+
+variable "lab_role_arn" {
+  description = "the role we use for all labs, dont use a single role for everything! it is an anti-pattern!!!!"
+  type = string
+
+}