s1201691/terraform
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Intitial state

Browse Source
This commit is contained in:
Wojciech Borowski-Dobrowolski 2024-05-12 07:31:59 +00:00
commit 8949ebc07f
9 changed files with 185 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
athena.tf Normal file
View File

@ -0,0 +1,27 @@
resource "aws_s3_bucket" "athena_results" {
bucket = "athena-results-${var.account_number}-${var.student_initials}-${var.student_index_no}"
force_destroy = true
tags = merge(local.common_tags)
}
resource "aws_s3_bucket_lifecycle_configuration" "athena_results_lifecycle" {
bucket = aws_s3_bucket.athena_results.id
rule {
id = "standard-expiration"
status = "Enabled"
expiration {
days=1
}
}
}
resource "aws_athena_workgroup" "athena_workgroup" {
name = "development"
configuration {
enforce_workgroup_configuration = true
result_configuration {
output_location = "s3://${aws_s3_bucket.athena_results.bucket}/output/"
}
}
force_destroy = true
}

17
glue_catalog_database.tf Normal file
View File

@ -0,0 +1,17 @@
resource "aws_glue_catalog_database" "datalake_db_raw_zone" {
name = "datalake_raw_${var.account_number}_${var.student_initials}_${var.student_index_no}"
}
resource "aws_glue_catalog_database" "datalake_db_processed_zone" {
name = "datalake_processed_${var.account_number}_${var.student_initials}_${var.student_index_no}"
}
resource "aws_glue_crawler" "glue_crawler_raw_zone" {
database_name = aws_glue_catalog_database.datalake_db_raw_zone.name
name = "gc-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
role = var.lab_role_arn
table_prefix = "crawler_"
s3_target {
path = "s3://${aws_s3_bucket.raw_bucket.bucket}/raw-zone/stockdata/"
}
tags = merge(local.common_tags, )
}

16
kinesis_ds.tf Normal file
View File

@ -0,0 +1,16 @@
resource "aws_kinesis_stream" "cryptostock_stream" {
name = "cryptostock-${var.account_number}-${var.student_initials}-${var.student_index_no}"
shard_count = 1
enforce_consumer_deletion = true
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
"IncomingRecords",
"OutgoingRecords"
]
tags = {
Purpose = "UAM Cloud Data Processing"
Environment = "DEV"
Owner = var.student_full_name
}
}

16
kinesis_fh.tf Normal file
View File

@ -0,0 +1,16 @@
resource "aws_kinesis_firehose_delivery_stream" "stock_delivery_stream" {
name = "firehose-${var.account_number}-${var.student_initials}-${var.student_index_no}"
destination = "extended_s3"
kinesis_source_configuration {
kinesis_stream_arn = aws_kinesis_stream.cryptostock_stream.arn
role_arn = var.lab_role_arn
}
extended_s3_configuration {
role_arn = var.lab_role_arn
bucket_arn = aws_s3_bucket.raw_bucket.arn
buffering_size = 1
buffering_interval = 60
prefix = "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
error_output_prefix = "${"raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}"}${"/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"}/"
}
}

36
lambda.tf Normal file
View File

@ -0,0 +1,36 @@
resource "aws_lambda_layer_version" "aws_wrangler" {
filename = "../lambda/awswrangler-layer-2.7.0-py3.8.zip"
layer_name = "aws_wrangler_${var.account_number}_${var.student_initials}_${var.student_index_no}"
source_code_hash = "${filebase64sha256("../lambda/awswrangler-layer-2.7.0-py3.8.zip")}"
compatible_runtimes = ["python3.8"]
}
resource "aws_lambda_function" "etl_post_processing" {
function_name = "etl-post-processing-${var.account_number}-${var.student_initials}-${var.student_index_no}"
filename = "lambda_definition.1.zip"
handler = "lambda_definition.etl_function"
runtime = "python3.8"
role = var.lab_role_arn
timeout = 300
memory_size = 512
source_code_hash= filebase64sha256("lambda_definition.1.zip")
layers = ["${aws_lambda_layer_version.aws_wrangler.arn}"]
}
resource "aws_lambda_permission" "allow_bucket" {
statement_id = "AllowExecutionFromS3Bucket"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.etl_post_processing.arn
principal = "s3.amazonaws.com"
source_arn = aws_s3_bucket.raw_bucket.arn
}
resource "aws_s3_bucket_notification" "trigger_etl_lambda" {
bucket = aws_s3_bucket.raw_bucket.id
lambda_function {
lambda_function_arn = aws_lambda_function.etl_post_processing.arn
events = ["s3:ObjectCreated:*"]
filter_prefix = "raw-zone/"
}
depends_on = [aws_lambda_permission.allow_bucket]
}

7
main.tf Normal file
View File

@ -0,0 +1,7 @@
locals {
common_tags = {
purpose = "UAM Cloud Data Processing"
environment = "DEV"
owner = var.student_full_name
}
}

13
provider.tf Normal file
View File

@ -0,0 +1,13 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 5.0"
}
}
}
provider "aws" {
profile = "default"
region = var.region
}

16
s3.tf Normal file
View File

@ -0,0 +1,16 @@
resource "aws_s3_bucket" "raw_bucket" {
bucket = "datalake-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
force_destroy = true
tags = {
Purpose = "UAM Cloud Data Processing"
Environment = "DEV"
}
}
resource "aws_s3_bucket" "processed_bucket" {
bucket = "datalake-processed-${var.account_number}-${var.student_initials}-${var.student_index_no}"
force_destroy = true
tags = {
Purpose = "UAM Cloud Data Processing"
Environment = "DEV"
}
}

37
variables.tf Normal file
View File

@ -0,0 +1,37 @@
variable "account_number" {
description = "Account number"
type = number
}
variable "region" {
description = "Region name - must be NVirginia us-east-1"
type = string
default = "us-east-1"
}
variable "environment" {
description = "Environment name"
type = string
default = "dev"
}
variable "student_initials" {
description = "letters of first and last names"
type = string
}
variable "student_full_name" {
description = "Student's full name"
type = string
}
variable "student_index_no" {
description = "Index no"
type = string
}
variable "lab_role_arn" {
description = "the role we use for all labs, dont use a single role for everything! it is an anti-pattern!!!!"
type = string
}