s1201680/chmury_aronb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added more terraform definitions

Browse Source
This commit is contained in:
EC2 Default User 2024-05-11 11:02:56 +00:00
parent bc7de6365e
commit 28d1f19042
6 changed files with 885 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
labs/sql_queries.sql Normal file
View File

@ -0,0 +1,12 @@
WITH cte AS(
SELECT DISTINCT date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H') AS HourlyBucket,
RANK() OVER(PARTITION BY (date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H')), symbol, type ORDER BY dollar_amount DESC) AS rnk,
transaction_ts, symbol, price, amount, dollar_amount, type, trans_id, year, month, day, hour
FROM "datalake_raw_534534002841_ab_1201680"."crawler_stockdata"
ORDER BY HourlyBucket
)
SELECT HourlyBucket, rnk, transaction_ts, symbol, price, amount, dollar_amount, type, trans_id, year, month, day, hour
FROM cte
WHERE rnk = 1
ORDER BY HourlyBucket, symbol, type

28
labs/terraform/athena.tf Normal file
View File

@ -0,0 +1,28 @@
resource "aws_s3_bucket" "athena_results" {
bucket = "athena-results-${var.account_number}-${var.student_initials}-${var.student_index_no}"
force_destroy = true
tags = merge(local.common_tags, )
}
resource "aws_s3_bucket_lifecycle_configuration" "athena_results_lifecycle" {
bucket = aws_s3_bucket.athena_results.id
rule {
id = "standard-expiration"
status = "Enabled"
expiration {
days=1
}
}
}
resource "aws_athena_workgroup" "athena_workgroup" {
name = "development"
configuration {
enforce_workgroup_configuration = true
result_configuration {
output_location = "s3://${aws_s3_bucket.athena_results.bucket}/output/"
}
}
force_destroy = true
}

16
labs/terraform/glue.tf Normal file
View File

@ -0,0 +1,16 @@
resource "aws_glue_catalog_database" "datalake_db_processed_zone" {
name = "datalake_processed_${var.account_number}_${var.student_initials}_${var.student_index_no}"
}
resource "aws_glue_crawler" "glue_crawler_raw_zone" {
database_name = aws_glue_catalog_database.datalake_db_raw_zone.name
name = "gc-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
role = var.lab_role_arn
table_prefix = "crawler_"
s3_target {
path = "s3://${aws_s3_bucket.raw_bucket.bucket}/raw-zone/stockdata/"
}
tags = merge(local.common_tags, )
}

36
labs/terraform/lambda.tf Normal file
View File

@ -0,0 +1,36 @@
resource "aws_lambda_layer_version" "aws_wrangler" {
filename = "../lambda/awswrangler-layer-2.7.0-py3.8.zip"
layer_name = "aws_wrangler_${var.account_number}_${var.student_initials}_${var.student_index_no}"
source_code_hash = "${filebase64sha256("../lambda/awswrangler-layer-2.7.0-py3.8.zip")}"
compatible_runtimes = ["python3.8"]
}
resource "aws_lambda_function" "etl_post_processing" {
function_name = "etl-post-processing-${var.account_number}-${var.student_initials}-${var.student_index_no}"
filename = "../lambda/lambda_definition.zip"
handler = "lambda_definition.etl_function"
runtime = "python3.8"
role = var.lab_role_arn
timeout = 300
memory_size = 512
source_code_hash= filebase64sha256("../lambda/lambda_definition.zip")
layers = ["${aws_lambda_layer_version.aws_wrangler.arn}"]
}
resource "aws_lambda_permission" "allow_bucket" {
statement_id = "AllowExecutionFromS3Bucket"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.etl_post_processing.arn
principal = "s3.amazonaws.com"
source_arn = aws_s3_bucket.raw_bucket.arn
}
resource "aws_s3_bucket_notification" "trigger_etl_lambda" {
bucket = aws_s3_bucket.raw_bucket.id
lambda_function {
lambda_function_arn = aws_lambda_function.etl_post_processing.arn
events = ["s3:ObjectCreated:*"]
filter_prefix = "raw-zone/"
}
depends_on = [aws_lambda_permission.allow_bucket]
}

322
labs/terraform/terraform.tfstate
View File

@ -1,327 +1,9 @@
{
"version": 4,
"terraform_version": "1.8.1",
"serial": 55,
"serial": 145,
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
"outputs": {},
"resources": [
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680",
"catalog_id": "534534002841",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "534534002841:datalake_raw_534534002841_ab_1201680",
"location_uri": "",
"name": "datalake_raw_534534002841_ab_1201680",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_firehose_delivery_stream",
"name": "stock_delivery_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"destination": "extended_s3",
"destination_id": "destinationId-000000000001",
"elasticsearch_configuration": [],
"extended_s3_configuration": [
{
"bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"buffering_interval": 60,
"buffering_size": 1,
"cloudwatch_logging_options": [
{
"enabled": false,
"log_group_name": "",
"log_stream_name": ""
}
],
"compression_format": "UNCOMPRESSED",
"custom_time_zone": "UTC",
"data_format_conversion_configuration": [],
"dynamic_partitioning_configuration": [],
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"file_extension": "",
"kms_key_arn": "",
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"processing_configuration": [
{
"enabled": false,
"processors": []
}
],
"role_arn": "arn:aws:iam::534534002841:role/LabRole",
"s3_backup_configuration": [],
"s3_backup_mode": "Disabled"
}
],
"http_endpoint_configuration": [],
"id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"kinesis_source_configuration": [
{
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"role_arn": "arn:aws:iam::534534002841:role/LabRole"
}
],
"msk_source_configuration": [],
"name": "firehose-534534002841-ab-1201680",
"opensearch_configuration": [],
"opensearchserverless_configuration": [],
"redshift_configuration": [],
"server_side_encryption": [
{
"enabled": false,
"key_arn": "",
"key_type": "AWS_OWNED_CMK"
}
],
"snowflake_configuration": [],
"splunk_configuration": [],
"tags": null,
"tags_all": {},
"timeouts": null,
"version_id": "1"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
"dependencies": [
"aws_kinesis_stream.cryptostock_stream",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_stream",
"name": "cryptostock_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"encryption_type": "NONE",
"enforce_consumer_deletion": true,
"id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"kms_key_id": "",
"name": "cryptostock-534534002841-ab-1201680",
"retention_period": 24,
"shard_count": 1,
"shard_level_metrics": [
"IncomingBytes",
"IncomingRecords",
"OutgoingBytes",
"OutgoingRecords"
],
"stream_mode_details": [
{
"stream_mode": "PROVISIONED"
}
],
"tags": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "processed_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680",
"bucket": "datalake-processed-534534002841-ab-1201680",
"bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-processed-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "raw_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"bucket": "datalake-raw-534534002841-ab-1201680",
"bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-raw-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
}
],
"resources": [],
"check_results": null
}

793
labs/terraform/terraform.tfstate.backup
View File

@ -1,9 +1,798 @@
{
"version": 4,
"terraform_version": "1.8.1",
"serial": 49,
"serial": 130,
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
"outputs": {},
"resources": [],
"resources": [
{
"mode": "managed",
"type": "aws_athena_workgroup",
"name": "athena_workgroup",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:athena:us-east-1:534534002841:workgroup/development",
"configuration": [
{
"bytes_scanned_cutoff_per_query": 0,
"enforce_workgroup_configuration": true,
"engine_version": [
{
"effective_engine_version": "Athena engine version 3",
"selected_engine_version": "AUTO"
}
],
"execution_role": "",
"publish_cloudwatch_metrics_enabled": true,
"requester_pays_enabled": false,
"result_configuration": [
{
"acl_configuration": [],
"encryption_configuration": [],
"expected_bucket_owner": "",
"output_location": "s3://athena-results-534534002841-ab-1201680/output/"
}
]
}
],
"description": "",
"force_destroy": true,
"id": "development",
"name": "development",
"state": "ENABLED",
"tags": null,
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_s3_bucket.athena_results"
]
}
]
},
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_processed_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_processed_534534002841_ab_1201680",
"catalog_id": "534534002841",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "534534002841:datalake_processed_534534002841_ab_1201680",
"location_uri": "",
"name": "datalake_processed_534534002841_ab_1201680",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680",
"catalog_id": "534534002841",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "534534002841:datalake_raw_534534002841_ab_1201680",
"location_uri": "",
"name": "datalake_raw_534534002841_ab_1201680",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_glue_crawler",
"name": "glue_crawler_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:534534002841:crawler/gc-raw-534534002841-ab-1201680",
"catalog_target": [],
"classifiers": null,
"configuration": "",
"database_name": "datalake_raw_534534002841_ab_1201680",
"delta_target": [],
"description": "",
"dynamodb_target": [],
"hudi_target": [],
"iceberg_target": [],
"id": "gc-raw-534534002841-ab-1201680",
"jdbc_target": [],
"lake_formation_configuration": [
{
"account_id": "",
"use_lake_formation_credentials": false
}
],
"lineage_configuration": [
{
"crawler_lineage_settings": "DISABLE"
}
],
"mongodb_target": [],
"name": "gc-raw-534534002841-ab-1201680",
"recrawl_policy": [
{
"recrawl_behavior": "CRAWL_EVERYTHING"
}
],
"role": "LabRole",
"s3_target": [
{
"connection_name": "",
"dlq_event_queue_arn": "",
"event_queue_arn": "",
"exclusions": null,
"path": "s3://datalake-raw-534534002841-ab-1201680/raw-zone/stockdata/",
"sample_size": 0
}
],
"schedule": "",
"schema_change_policy": [
{
"delete_behavior": "DEPRECATE_IN_DATABASE",
"update_behavior": "UPDATE_IN_DATABASE"
}
],
"security_configuration": "",
"table_prefix": "crawler_",
"tags": {
"environment": "DEV",
"owner": "Aron Boguszewski",
"purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"environment": "DEV",
"owner": "Aron Boguszewski",
"purpose": "UAM Cloud Data Processing"
}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_glue_catalog_database.datalake_db_raw_zone",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_firehose_delivery_stream",
"name": "stock_delivery_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"destination": "extended_s3",
"destination_id": "destinationId-000000000001",
"elasticsearch_configuration": [],
"extended_s3_configuration": [
{
"bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"buffering_interval": 60,
"buffering_size": 1,
"cloudwatch_logging_options": [
{
"enabled": false,
"log_group_name": "",
"log_stream_name": ""
}
],
"compression_format": "UNCOMPRESSED",
"custom_time_zone": "UTC",
"data_format_conversion_configuration": [],
"dynamic_partitioning_configuration": [],
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"file_extension": "",
"kms_key_arn": "",
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"processing_configuration": [
{
"enabled": false,
"processors": []
}
],
"role_arn": "arn:aws:iam::534534002841:role/LabRole",
"s3_backup_configuration": [],
"s3_backup_mode": "Disabled"
}
],
"http_endpoint_configuration": [],
"id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"kinesis_source_configuration": [
{
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"role_arn": "arn:aws:iam::534534002841:role/LabRole"
}
],
"msk_source_configuration": [],
"name": "firehose-534534002841-ab-1201680",
"opensearch_configuration": [],
"opensearchserverless_configuration": [],
"redshift_configuration": [],
"server_side_encryption": [
{
"enabled": false,
"key_arn": "",
"key_type": "AWS_OWNED_CMK"
}
],
"snowflake_configuration": [],
"splunk_configuration": [],
"tags": null,
"tags_all": {},
"timeouts": null,
"version_id": "1"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
"dependencies": [
"aws_kinesis_stream.cryptostock_stream",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_stream",
"name": "cryptostock_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"encryption_type": "NONE",
"enforce_consumer_deletion": true,
"id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"kms_key_id": "",
"name": "cryptostock-534534002841-ab-1201680",
"retention_period": 24,
"shard_count": 1,
"shard_level_metrics": [
"IncomingBytes",
"IncomingRecords",
"OutgoingBytes",
"OutgoingRecords"
],
"stream_mode_details": [
{
"stream_mode": "PROVISIONED"
}
],
"tags": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_lambda_function",
"name": "etl_post_processing",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"architectures": [
"x86_64"
],
"arn": "arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680",
"code_signing_config_arn": "",
"dead_letter_config": [],
"description": "",
"environment": [],
"ephemeral_storage": [
{
"size": 512
}
],
"file_system_config": [],
"filename": "../lambda/lambda_definition.zip",
"function_name": "etl-post-processing-534534002841-ab-1201680",
"handler": "lambda_definition.etl_function",
"id": "etl-post-processing-534534002841-ab-1201680",
"image_config": [],
"image_uri": "",
"invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680/invocations",
"kms_key_arn": "",
"last_modified": "2024-05-11T10:53:43.594+0000",
"layers": [
"arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680:2"
],
"logging_config": [
{
"application_log_level": "",
"log_format": "Text",
"log_group": "/aws/lambda/etl-post-processing-534534002841-ab-1201680",
"system_log_level": ""
}
],
"memory_size": 512,
"package_type": "Zip",
"publish": false,
"qualified_arn": "arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680:$LATEST",
"qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680:$LATEST/invocations",
"replace_security_groups_on_destroy": null,
"replacement_security_group_ids": null,
"reserved_concurrent_executions": -1,
"role": "arn:aws:iam::534534002841:role/LabRole",
"runtime": "python3.8",
"s3_bucket": null,
"s3_key": null,
"s3_object_version": null,
"signing_job_arn": "",
"signing_profile_version_arn": "",
"skip_destroy": false,
"snap_start": [],
"source_code_hash": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=",
"source_code_size": 884,
"tags": null,
"tags_all": {},
"timeout": 300,
"timeouts": null,
"tracing_config": [
{
"mode": "PassThrough"
}
],
"version": "$LATEST",
"vpc_config": []
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19",
"dependencies": [
"aws_lambda_layer_version.aws_wrangler"
]
}
]
},
{
"mode": "managed",
"type": "aws_lambda_layer_version",
"name": "aws_wrangler",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680:2",
"compatible_architectures": null,
"compatible_runtimes": [
"python3.8"
],
"created_date": "2024-05-11T10:53:43.233+0000",
"description": "",
"filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip",
"id": "arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680:2",
"layer_arn": "arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680",
"layer_name": "aws_wrangler_534534002841_ab_1201680",
"license_info": "",
"s3_bucket": null,
"s3_key": null,
"s3_object_version": null,
"signing_job_arn": "",
"signing_profile_version_arn": "",
"skip_destroy": false,
"source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=",
"source_code_size": 43879070,
"version": "2"
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_lambda_permission",
"name": "allow_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"action": "lambda:InvokeFunction",
"event_source_token": null,
"function_name": "arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680",
"function_url_auth_type": null,
"id": "AllowExecutionFromS3Bucket",
"principal": "s3.amazonaws.com",
"principal_org_id": null,
"qualifier": "",
"source_account": null,
"source_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"statement_id": "AllowExecutionFromS3Bucket",
"statement_id_prefix": ""
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_lambda_function.etl_post_processing",
"aws_lambda_layer_version.aws_wrangler",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "athena_results",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::athena-results-534534002841-ab-1201680",
"bucket": "athena-results-534534002841-ab-1201680",
"bucket_domain_name": "athena-results-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "athena-results-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "athena-results-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"environment": "DEV",
"owner": "Aron Boguszewski",
"purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"environment": "DEV",
"owner": "Aron Boguszewski",
"purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "processed_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680",
"bucket": "datalake-processed-534534002841-ab-1201680",
"bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-processed-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "raw_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"bucket": "datalake-raw-534534002841-ab-1201680",
"bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-raw-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket_lifecycle_configuration",
"name": "athena_results_lifecycle",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"bucket": "athena-results-534534002841-ab-1201680",
"expected_bucket_owner": "",
"id": "athena-results-534534002841-ab-1201680",
"rule": [
{
"abort_incomplete_multipart_upload": [],
"expiration": [
{
"date": "",
"days": 1,
"expired_object_delete_marker": false
}
],
"filter": [
{
"and": [],
"object_size_greater_than": "",
"object_size_less_than": "",
"prefix": "",
"tag": []
}
],
"id": "standard-expiration",
"noncurrent_version_expiration": [],
"noncurrent_version_transition": [],
"prefix": "",
"status": "Enabled",
"transition": []
}
],
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=",
"dependencies": [
"aws_s3_bucket.athena_results"
]
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket_notification",
"name": "trigger_etl_lambda",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"bucket": "datalake-raw-534534002841-ab-1201680",
"eventbridge": false,
"id": "datalake-raw-534534002841-ab-1201680",
"lambda_function": [],
"queue": [],
"topic": []
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_lambda_function.etl_post_processing",
"aws_lambda_layer_version.aws_wrangler",
"aws_lambda_permission.allow_bucket",
"aws_s3_bucket.raw_bucket"
]
}
]
}
],
"check_results": null
}