Intitial state
This commit is contained in:
commit
8949ebc07f
|
|
@ -0,0 +1,27 @@
|
|||
resource "aws_s3_bucket" "athena_results" {
|
||||
bucket = "athena-results-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||
force_destroy = true
|
||||
tags = merge(local.common_tags)
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_lifecycle_configuration" "athena_results_lifecycle" {
|
||||
bucket = aws_s3_bucket.athena_results.id
|
||||
rule {
|
||||
id = "standard-expiration"
|
||||
status = "Enabled"
|
||||
expiration {
|
||||
days=1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_athena_workgroup" "athena_workgroup" {
|
||||
name = "development"
|
||||
configuration {
|
||||
enforce_workgroup_configuration = true
|
||||
result_configuration {
|
||||
output_location = "s3://${aws_s3_bucket.athena_results.bucket}/output/"
|
||||
}
|
||||
}
|
||||
force_destroy = true
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
resource "aws_glue_catalog_database" "datalake_db_raw_zone" {
|
||||
name = "datalake_raw_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
||||
}
|
||||
|
||||
resource "aws_glue_catalog_database" "datalake_db_processed_zone" {
|
||||
name = "datalake_processed_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
||||
}
|
||||
resource "aws_glue_crawler" "glue_crawler_raw_zone" {
|
||||
database_name = aws_glue_catalog_database.datalake_db_raw_zone.name
|
||||
name = "gc-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||
role = var.lab_role_arn
|
||||
table_prefix = "crawler_"
|
||||
s3_target {
|
||||
path = "s3://${aws_s3_bucket.raw_bucket.bucket}/raw-zone/stockdata/"
|
||||
}
|
||||
tags = merge(local.common_tags, )
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
resource "aws_kinesis_stream" "cryptostock_stream" {
|
||||
name = "cryptostock-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||
shard_count = 1
|
||||
enforce_consumer_deletion = true
|
||||
shard_level_metrics = [
|
||||
"IncomingBytes",
|
||||
"OutgoingBytes",
|
||||
"IncomingRecords",
|
||||
"OutgoingRecords"
|
||||
]
|
||||
tags = {
|
||||
Purpose = "UAM Cloud Data Processing"
|
||||
Environment = "DEV"
|
||||
Owner = var.student_full_name
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
resource "aws_kinesis_firehose_delivery_stream" "stock_delivery_stream" {
|
||||
name = "firehose-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||
destination = "extended_s3"
|
||||
kinesis_source_configuration {
|
||||
kinesis_stream_arn = aws_kinesis_stream.cryptostock_stream.arn
|
||||
role_arn = var.lab_role_arn
|
||||
}
|
||||
extended_s3_configuration {
|
||||
role_arn = var.lab_role_arn
|
||||
bucket_arn = aws_s3_bucket.raw_bucket.arn
|
||||
buffering_size = 1
|
||||
buffering_interval = 60
|
||||
prefix = "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
|
||||
error_output_prefix = "${"raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}"}${"/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"}/"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
resource "aws_lambda_layer_version" "aws_wrangler" {
|
||||
filename = "../lambda/awswrangler-layer-2.7.0-py3.8.zip"
|
||||
layer_name = "aws_wrangler_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
||||
source_code_hash = "${filebase64sha256("../lambda/awswrangler-layer-2.7.0-py3.8.zip")}"
|
||||
compatible_runtimes = ["python3.8"]
|
||||
}
|
||||
|
||||
resource "aws_lambda_function" "etl_post_processing" {
|
||||
function_name = "etl-post-processing-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||
filename = "lambda_definition.1.zip"
|
||||
handler = "lambda_definition.etl_function"
|
||||
runtime = "python3.8"
|
||||
role = var.lab_role_arn
|
||||
timeout = 300
|
||||
memory_size = 512
|
||||
source_code_hash= filebase64sha256("lambda_definition.1.zip")
|
||||
layers = ["${aws_lambda_layer_version.aws_wrangler.arn}"]
|
||||
}
|
||||
|
||||
resource "aws_lambda_permission" "allow_bucket" {
|
||||
statement_id = "AllowExecutionFromS3Bucket"
|
||||
action = "lambda:InvokeFunction"
|
||||
function_name = aws_lambda_function.etl_post_processing.arn
|
||||
principal = "s3.amazonaws.com"
|
||||
source_arn = aws_s3_bucket.raw_bucket.arn
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_notification" "trigger_etl_lambda" {
|
||||
bucket = aws_s3_bucket.raw_bucket.id
|
||||
lambda_function {
|
||||
lambda_function_arn = aws_lambda_function.etl_post_processing.arn
|
||||
events = ["s3:ObjectCreated:*"]
|
||||
filter_prefix = "raw-zone/"
|
||||
}
|
||||
depends_on = [aws_lambda_permission.allow_bucket]
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
locals {
|
||||
common_tags = {
|
||||
purpose = "UAM Cloud Data Processing"
|
||||
environment = "DEV"
|
||||
owner = var.student_full_name
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
profile = "default"
|
||||
region = var.region
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
resource "aws_s3_bucket" "raw_bucket" {
|
||||
bucket = "datalake-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||
force_destroy = true
|
||||
tags = {
|
||||
Purpose = "UAM Cloud Data Processing"
|
||||
Environment = "DEV"
|
||||
}
|
||||
}
|
||||
resource "aws_s3_bucket" "processed_bucket" {
|
||||
bucket = "datalake-processed-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||
force_destroy = true
|
||||
tags = {
|
||||
Purpose = "UAM Cloud Data Processing"
|
||||
Environment = "DEV"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
variable "account_number" {
|
||||
description = "Account number"
|
||||
type = number
|
||||
}
|
||||
|
||||
variable "region" {
|
||||
description = "Region name - must be NVirginia us-east-1"
|
||||
type = string
|
||||
default = "us-east-1"
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
description = "Environment name"
|
||||
type = string
|
||||
default = "dev"
|
||||
}
|
||||
|
||||
variable "student_initials" {
|
||||
description = "letters of first and last names"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "student_full_name" {
|
||||
description = "Student's full name"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "student_index_no" {
|
||||
description = "Index no"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "lab_role_arn" {
|
||||
description = "the role we use for all labs, dont use a single role for everything! it is an anti-pattern!!!!"
|
||||
type = string
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue