zadania
This commit is contained in:
parent
c543423de6
commit
8d6b3978cd
35
labs/query_sql.sql
Normal file
35
labs/query_sql.sql
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
WITH cte AS(
|
||||||
|
SELECT
|
||||||
|
DISTINCT date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H') AS HourlyBucket,
|
||||||
|
RANK() OVER(PARTITION BY (date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H')), symbol, type ORDER BY dollar_amount DESC) AS ranking,
|
||||||
|
transaction_ts,
|
||||||
|
symbol,
|
||||||
|
price,
|
||||||
|
amount,
|
||||||
|
dollar_amount,
|
||||||
|
type,
|
||||||
|
trans_id,
|
||||||
|
year,
|
||||||
|
month,
|
||||||
|
day,
|
||||||
|
hour
|
||||||
|
FROM "datalake_raw_613161579745_rc_1201682"."crawler_stockdata"
|
||||||
|
ORDER BY HourlyBucket
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
HourlyBucket,
|
||||||
|
ranking,
|
||||||
|
transaction_ts,
|
||||||
|
symbol,
|
||||||
|
price,
|
||||||
|
amount,
|
||||||
|
dollar_amount,
|
||||||
|
type,
|
||||||
|
trans_id,
|
||||||
|
year,
|
||||||
|
month,
|
||||||
|
day,
|
||||||
|
hour
|
||||||
|
FROM cte
|
||||||
|
WHERE ranking = 1
|
||||||
|
ORDER BY HourlyBucket, symbol, type
|
28
labs/terraform/athena.tf
Normal file
28
labs/terraform/athena.tf
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
resource "aws_s3_bucket" "athena_results" {
|
||||||
|
bucket = "athena-results-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||||
|
force_destroy = true
|
||||||
|
tags = merge(local.common_tags, )
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_lifecycle_configuration" "athena_results_lifecycle" {
|
||||||
|
bucket = aws_s3_bucket.athena_results.id
|
||||||
|
|
||||||
|
rule {
|
||||||
|
id = "standard-expiration"
|
||||||
|
status = "Enabled"
|
||||||
|
expiration {
|
||||||
|
days=1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_athena_workgroup" "athena_workgroup" {
|
||||||
|
name = "development"
|
||||||
|
configuration {
|
||||||
|
enforce_workgroup_configuration = true
|
||||||
|
result_configuration {
|
||||||
|
output_location = "s3://${aws_s3_bucket.athena_results.bucket}/output/"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
force_destroy = true
|
||||||
|
}
|
16
labs/terraform/glue.tf
Normal file
16
labs/terraform/glue.tf
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
resource "aws_glue_catalog_database" "datalake_db_processed_zone" {
|
||||||
|
name = "datalake_processed_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_glue_crawler" "glue_crawler_raw_zone" {
|
||||||
|
database_name = aws_glue_catalog_database.datalake_db_raw_zone.name
|
||||||
|
name = "gc-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||||
|
role = var.lab_role_arn
|
||||||
|
table_prefix = "crawler_"
|
||||||
|
|
||||||
|
s3_target {
|
||||||
|
path = "s3://${aws_s3_bucket.raw_bucket.bucket}/raw-zone/stockdata/"
|
||||||
|
}
|
||||||
|
|
||||||
|
tags = merge(local.common_tags, )
|
||||||
|
}
|
3
labs/terraform/glue_dc.tf
Normal file
3
labs/terraform/glue_dc.tf
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
resource "aws_glue_catalog_database" "datalake_db_raw_zone" {
|
||||||
|
name = "datalake_raw_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
||||||
|
}
|
16
labs/terraform/kinesis.tf
Normal file
16
labs/terraform/kinesis.tf
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
resource "aws_kinesis_stream" "cryptostock_stream" {
|
||||||
|
name = "cryptostock-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||||
|
shard_count = 1
|
||||||
|
enforce_consumer_deletion = true
|
||||||
|
shard_level_metrics = [
|
||||||
|
"IncomingBytes",
|
||||||
|
"OutgoingBytes",
|
||||||
|
"IncomingRecords",
|
||||||
|
"OutgoingRecords"
|
||||||
|
]
|
||||||
|
tags = {
|
||||||
|
Purpose = "UAM Cloud Data Processing"
|
||||||
|
Environment = "DEV"
|
||||||
|
Owner = var.student_full_name
|
||||||
|
}
|
||||||
|
}
|
16
labs/terraform/kinesis_fh.tf
Normal file
16
labs/terraform/kinesis_fh.tf
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
resource "aws_kinesis_firehose_delivery_stream" "stock_delivery_stream"{
|
||||||
|
name ="firehose-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||||
|
destination = "extended_s3"
|
||||||
|
kinesis_source_configuration {
|
||||||
|
kinesis_stream_arn = aws_kinesis_stream.cryptostock_stream.arn
|
||||||
|
role_arn = var.lab_role_arn
|
||||||
|
}
|
||||||
|
extended_s3_configuration {
|
||||||
|
role_arn = var.lab_role_arn
|
||||||
|
bucket_arn = aws_s3_bucket.raw_bucket.arn
|
||||||
|
buffering_size = 1
|
||||||
|
buffering_interval = 60
|
||||||
|
prefix ="raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
|
||||||
|
error_output_prefix ="${"raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}"}${"/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"}/"
|
||||||
|
}
|
||||||
|
}
|
36
labs/terraform/lambda.tf
Normal file
36
labs/terraform/lambda.tf
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
resource "aws_lambda_layer_version" "aws_wrangler" {
|
||||||
|
filename = "../lambda/awswrangler-layer-2.7.0-py3.8.zip"
|
||||||
|
layer_name = "aws_wrangler_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
||||||
|
source_code_hash = "${filebase64sha256("../lambda/awswrangler-layer-2.7.0-py3.8.zip")}"
|
||||||
|
compatible_runtimes = ["python3.8"]
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_lambda_function" "etl_post_processing" {
|
||||||
|
function_name = "etl-post-processing-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||||
|
filename = "../lambda/lambda_definition.zip"
|
||||||
|
handler = "lambda_definition.etl_function"
|
||||||
|
runtime = "python3.8"
|
||||||
|
role = var.lab_role_arn
|
||||||
|
timeout = 300
|
||||||
|
memory_size = 512
|
||||||
|
source_code_hash= filebase64sha256("../lambda/lambda_definition.zip")
|
||||||
|
layers = ["${aws_lambda_layer_version.aws_wrangler.arn}"]
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_lambda_permission" "allow_bucket" {
|
||||||
|
statement_id = "AllowExecutionFromS3Bucket"
|
||||||
|
action = "lambda:InvokeFunction"
|
||||||
|
function_name = aws_lambda_function.etl_post_processing.arn
|
||||||
|
principal = "s3.amazonaws.com"
|
||||||
|
source_arn = aws_s3_bucket.raw_bucket.arn
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_notification" "trigger_etl_lambda" {
|
||||||
|
bucket = aws_s3_bucket.raw_bucket.id
|
||||||
|
lambda_function {
|
||||||
|
lambda_function_arn = aws_lambda_function.etl_post_processing.arn
|
||||||
|
events = ["s3:ObjectCreated:*"]
|
||||||
|
filter_prefix = "raw-zone/"
|
||||||
|
}
|
||||||
|
depends_on = [aws_lambda_permission.allow_bucket]
|
||||||
|
}
|
19
labs/terraform/s3.tf
Normal file
19
labs/terraform/s3.tf
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
resource "aws_s3_bucket" "raw_bucket" {
|
||||||
|
bucket ="datalake-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||||
|
force_destroy = true
|
||||||
|
tags = {
|
||||||
|
Purpose = "UAM Cloud Data Processing"
|
||||||
|
Environment = "DEV"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket" "processed_bucket" {
|
||||||
|
bucket ="datalake-processed-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
||||||
|
force_destroy = true
|
||||||
|
tags = {
|
||||||
|
Purpose = "UAM Cloud Data Processing"
|
||||||
|
Environment = "PROD"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
9
labs/terraform/terraform.tfstate
Normal file
9
labs/terraform/terraform.tfstate
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"version": 4,
|
||||||
|
"terraform_version": "1.8.1",
|
||||||
|
"serial": 68,
|
||||||
|
"lineage": "a7afa363-ceb3-3da5-99b5-5c54dec39deb",
|
||||||
|
"outputs": {},
|
||||||
|
"resources": [],
|
||||||
|
"check_results": null
|
||||||
|
}
|
810
labs/terraform/terraform.tfstate.backup
Normal file
810
labs/terraform/terraform.tfstate.backup
Normal file
@ -0,0 +1,810 @@
|
|||||||
|
{
|
||||||
|
"version": 4,
|
||||||
|
"terraform_version": "1.8.1",
|
||||||
|
"serial": 53,
|
||||||
|
"lineage": "a7afa363-ceb3-3da5-99b5-5c54dec39deb",
|
||||||
|
"outputs": {},
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_athena_workgroup",
|
||||||
|
"name": "athena_workgroup",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:athena:us-east-1:613161579745:workgroup/development",
|
||||||
|
"configuration": [
|
||||||
|
{
|
||||||
|
"bytes_scanned_cutoff_per_query": 0,
|
||||||
|
"enforce_workgroup_configuration": true,
|
||||||
|
"engine_version": [
|
||||||
|
{
|
||||||
|
"effective_engine_version": "Athena engine version 3",
|
||||||
|
"selected_engine_version": "AUTO"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"execution_role": "",
|
||||||
|
"publish_cloudwatch_metrics_enabled": true,
|
||||||
|
"requester_pays_enabled": false,
|
||||||
|
"result_configuration": [
|
||||||
|
{
|
||||||
|
"acl_configuration": [],
|
||||||
|
"encryption_configuration": [],
|
||||||
|
"expected_bucket_owner": "",
|
||||||
|
"output_location": "s3://athena-results-613161579745-rc-s1201682/output/"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"force_destroy": true,
|
||||||
|
"id": "development",
|
||||||
|
"name": "development",
|
||||||
|
"state": "ENABLED",
|
||||||
|
"tags": null,
|
||||||
|
"tags_all": {}
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA==",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_s3_bucket.athena_results"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_glue_catalog_database",
|
||||||
|
"name": "datalake_db_processed_zone",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:glue:us-east-1:613161579745:database/datalake_processed_613161579745_rc_s1201682",
|
||||||
|
"catalog_id": "613161579745",
|
||||||
|
"create_table_default_permission": [
|
||||||
|
{
|
||||||
|
"permissions": [
|
||||||
|
"ALL"
|
||||||
|
],
|
||||||
|
"principal": [
|
||||||
|
{
|
||||||
|
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"federated_database": [],
|
||||||
|
"id": "613161579745:datalake_processed_613161579745_rc_s1201682",
|
||||||
|
"location_uri": "",
|
||||||
|
"name": "datalake_processed_613161579745_rc_s1201682",
|
||||||
|
"parameters": null,
|
||||||
|
"tags": null,
|
||||||
|
"tags_all": {},
|
||||||
|
"target_database": []
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA=="
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_glue_catalog_database",
|
||||||
|
"name": "datalake_db_raw_zone",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:glue:us-east-1:613161579745:database/datalake_raw_613161579745_rc_s1201682",
|
||||||
|
"catalog_id": "613161579745",
|
||||||
|
"create_table_default_permission": [
|
||||||
|
{
|
||||||
|
"permissions": [
|
||||||
|
"ALL"
|
||||||
|
],
|
||||||
|
"principal": [
|
||||||
|
{
|
||||||
|
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"federated_database": [],
|
||||||
|
"id": "613161579745:datalake_raw_613161579745_rc_s1201682",
|
||||||
|
"location_uri": "",
|
||||||
|
"name": "datalake_raw_613161579745_rc_s1201682",
|
||||||
|
"parameters": {},
|
||||||
|
"tags": {},
|
||||||
|
"tags_all": {},
|
||||||
|
"target_database": []
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA=="
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_glue_crawler",
|
||||||
|
"name": "glue_crawler_raw_zone",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:glue:us-east-1:613161579745:crawler/gc-raw-613161579745-rc-s1201682",
|
||||||
|
"catalog_target": [],
|
||||||
|
"classifiers": null,
|
||||||
|
"configuration": "",
|
||||||
|
"database_name": "datalake_raw_613161579745_rc_s1201682",
|
||||||
|
"delta_target": [],
|
||||||
|
"description": "",
|
||||||
|
"dynamodb_target": [],
|
||||||
|
"hudi_target": [],
|
||||||
|
"iceberg_target": [],
|
||||||
|
"id": "gc-raw-613161579745-rc-s1201682",
|
||||||
|
"jdbc_target": [],
|
||||||
|
"lake_formation_configuration": [
|
||||||
|
{
|
||||||
|
"account_id": "",
|
||||||
|
"use_lake_formation_credentials": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"lineage_configuration": [
|
||||||
|
{
|
||||||
|
"crawler_lineage_settings": "DISABLE"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"mongodb_target": [],
|
||||||
|
"name": "gc-raw-613161579745-rc-s1201682",
|
||||||
|
"recrawl_policy": [
|
||||||
|
{
|
||||||
|
"recrawl_behavior": "CRAWL_EVERYTHING"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"role": "LabRole",
|
||||||
|
"s3_target": [
|
||||||
|
{
|
||||||
|
"connection_name": "",
|
||||||
|
"dlq_event_queue_arn": "",
|
||||||
|
"event_queue_arn": "",
|
||||||
|
"exclusions": null,
|
||||||
|
"path": "s3://datalake-raw-613161579745-rc-s1201682/raw-zone/stockdata/",
|
||||||
|
"sample_size": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schedule": "",
|
||||||
|
"schema_change_policy": [
|
||||||
|
{
|
||||||
|
"delete_behavior": "DEPRECATE_IN_DATABASE",
|
||||||
|
"update_behavior": "UPDATE_IN_DATABASE"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"security_configuration": "",
|
||||||
|
"table_prefix": "crawler_",
|
||||||
|
"tags": {
|
||||||
|
"environment": "DEV",
|
||||||
|
"owner": "Rafal Charzewski",
|
||||||
|
"purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"environment": "DEV",
|
||||||
|
"owner": "Rafal Charzewski",
|
||||||
|
"purpose": "UAM Cloud Data Processing"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA==",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_glue_catalog_database.datalake_db_raw_zone",
|
||||||
|
"aws_s3_bucket.raw_bucket"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_kinesis_firehose_delivery_stream",
|
||||||
|
"name": "stock_delivery_stream",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:firehose:us-east-1:613161579745:deliverystream/firehose-613161579745-rc-s1201682",
|
||||||
|
"destination": "extended_s3",
|
||||||
|
"destination_id": "destinationId-000000000001",
|
||||||
|
"elasticsearch_configuration": [],
|
||||||
|
"extended_s3_configuration": [
|
||||||
|
{
|
||||||
|
"bucket_arn": "arn:aws:s3:::datalake-raw-613161579745-rc-s1201682",
|
||||||
|
"buffering_interval": 60,
|
||||||
|
"buffering_size": 1,
|
||||||
|
"cloudwatch_logging_options": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"log_group_name": "",
|
||||||
|
"log_stream_name": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"compression_format": "UNCOMPRESSED",
|
||||||
|
"custom_time_zone": "UTC",
|
||||||
|
"data_format_conversion_configuration": [],
|
||||||
|
"dynamic_partitioning_configuration": [],
|
||||||
|
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||||
|
"file_extension": "",
|
||||||
|
"kms_key_arn": "",
|
||||||
|
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||||
|
"processing_configuration": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"processors": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"role_arn": "arn:aws:iam::613161579745:role/LabRole",
|
||||||
|
"s3_backup_configuration": [],
|
||||||
|
"s3_backup_mode": "Disabled"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"http_endpoint_configuration": [],
|
||||||
|
"id": "arn:aws:firehose:us-east-1:613161579745:deliverystream/firehose-613161579745-rc-s1201682",
|
||||||
|
"kinesis_source_configuration": [
|
||||||
|
{
|
||||||
|
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:613161579745:stream/cryptostock-613161579745-rc-s1201682",
|
||||||
|
"role_arn": "arn:aws:iam::613161579745:role/LabRole"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"msk_source_configuration": [],
|
||||||
|
"name": "firehose-613161579745-rc-s1201682",
|
||||||
|
"opensearch_configuration": [],
|
||||||
|
"opensearchserverless_configuration": [],
|
||||||
|
"redshift_configuration": [],
|
||||||
|
"server_side_encryption": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"key_arn": "",
|
||||||
|
"key_type": "AWS_OWNED_CMK"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"snowflake_configuration": [],
|
||||||
|
"splunk_configuration": [],
|
||||||
|
"tags": {},
|
||||||
|
"tags_all": {},
|
||||||
|
"timeouts": null,
|
||||||
|
"version_id": "1"
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_kinesis_stream.cryptostock_stream",
|
||||||
|
"aws_s3_bucket.raw_bucket"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_kinesis_stream",
|
||||||
|
"name": "cryptostock_stream",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:kinesis:us-east-1:613161579745:stream/cryptostock-613161579745-rc-s1201682",
|
||||||
|
"encryption_type": "NONE",
|
||||||
|
"enforce_consumer_deletion": true,
|
||||||
|
"id": "arn:aws:kinesis:us-east-1:613161579745:stream/cryptostock-613161579745-rc-s1201682",
|
||||||
|
"kms_key_id": "",
|
||||||
|
"name": "cryptostock-613161579745-rc-s1201682",
|
||||||
|
"retention_period": 24,
|
||||||
|
"shard_count": 1,
|
||||||
|
"shard_level_metrics": [
|
||||||
|
"IncomingBytes",
|
||||||
|
"IncomingRecords",
|
||||||
|
"OutgoingBytes",
|
||||||
|
"OutgoingRecords"
|
||||||
|
],
|
||||||
|
"stream_mode_details": [
|
||||||
|
{
|
||||||
|
"stream_mode": "PROVISIONED"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Owner": "Rafal Charzewski",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Owner": "Rafal Charzewski",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"timeouts": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_lambda_function",
|
||||||
|
"name": "etl_post_processing",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"architectures": [
|
||||||
|
"x86_64"
|
||||||
|
],
|
||||||
|
"arn": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682",
|
||||||
|
"code_sha256": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=",
|
||||||
|
"code_signing_config_arn": "",
|
||||||
|
"dead_letter_config": [],
|
||||||
|
"description": "",
|
||||||
|
"environment": [],
|
||||||
|
"ephemeral_storage": [
|
||||||
|
{
|
||||||
|
"size": 512
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"file_system_config": [],
|
||||||
|
"filename": "../lambda/lambda_definition.zip",
|
||||||
|
"function_name": "etl-post-processing-613161579745-rc-s1201682",
|
||||||
|
"handler": "lambda_definition.etl_function",
|
||||||
|
"id": "etl-post-processing-613161579745-rc-s1201682",
|
||||||
|
"image_config": [],
|
||||||
|
"image_uri": "",
|
||||||
|
"invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682/invocations",
|
||||||
|
"kms_key_arn": "",
|
||||||
|
"last_modified": "2024-06-19T22:46:02.439+0000",
|
||||||
|
"layers": [
|
||||||
|
"arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682:1"
|
||||||
|
],
|
||||||
|
"logging_config": [
|
||||||
|
{
|
||||||
|
"application_log_level": "",
|
||||||
|
"log_format": "Text",
|
||||||
|
"log_group": "/aws/lambda/etl-post-processing-613161579745-rc-s1201682",
|
||||||
|
"system_log_level": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"memory_size": 512,
|
||||||
|
"package_type": "Zip",
|
||||||
|
"publish": false,
|
||||||
|
"qualified_arn": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682:$LATEST",
|
||||||
|
"qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682:$LATEST/invocations",
|
||||||
|
"replace_security_groups_on_destroy": null,
|
||||||
|
"replacement_security_group_ids": null,
|
||||||
|
"reserved_concurrent_executions": -1,
|
||||||
|
"role": "arn:aws:iam::613161579745:role/LabRole",
|
||||||
|
"runtime": "python3.8",
|
||||||
|
"s3_bucket": null,
|
||||||
|
"s3_key": null,
|
||||||
|
"s3_object_version": null,
|
||||||
|
"signing_job_arn": "",
|
||||||
|
"signing_profile_version_arn": "",
|
||||||
|
"skip_destroy": false,
|
||||||
|
"snap_start": [],
|
||||||
|
"source_code_hash": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=",
|
||||||
|
"source_code_size": 884,
|
||||||
|
"tags": null,
|
||||||
|
"tags_all": {},
|
||||||
|
"timeout": 300,
|
||||||
|
"timeouts": null,
|
||||||
|
"tracing_config": [
|
||||||
|
{
|
||||||
|
"mode": "PassThrough"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"version": "$LATEST",
|
||||||
|
"vpc_config": []
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_lambda_layer_version.aws_wrangler"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_lambda_layer_version",
|
||||||
|
"name": "aws_wrangler",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682:1",
|
||||||
|
"code_sha256": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=",
|
||||||
|
"compatible_architectures": null,
|
||||||
|
"compatible_runtimes": [
|
||||||
|
"python3.8"
|
||||||
|
],
|
||||||
|
"created_date": "2024-06-19T22:46:02.186+0000",
|
||||||
|
"description": "",
|
||||||
|
"filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip",
|
||||||
|
"id": "arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682:1",
|
||||||
|
"layer_arn": "arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682",
|
||||||
|
"layer_name": "aws_wrangler_613161579745_rc_s1201682",
|
||||||
|
"license_info": "",
|
||||||
|
"s3_bucket": null,
|
||||||
|
"s3_key": null,
|
||||||
|
"s3_object_version": null,
|
||||||
|
"signing_job_arn": "",
|
||||||
|
"signing_profile_version_arn": "",
|
||||||
|
"skip_destroy": false,
|
||||||
|
"source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=",
|
||||||
|
"source_code_size": 43879070,
|
||||||
|
"version": "1"
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA=="
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_lambda_permission",
|
||||||
|
"name": "allow_bucket",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"action": "lambda:InvokeFunction",
|
||||||
|
"event_source_token": null,
|
||||||
|
"function_name": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682",
|
||||||
|
"function_url_auth_type": null,
|
||||||
|
"id": "AllowExecutionFromS3Bucket",
|
||||||
|
"principal": "s3.amazonaws.com",
|
||||||
|
"principal_org_id": null,
|
||||||
|
"qualifier": "",
|
||||||
|
"source_account": null,
|
||||||
|
"source_arn": "arn:aws:s3:::datalake-raw-613161579745-rc-s1201682",
|
||||||
|
"statement_id": "AllowExecutionFromS3Bucket",
|
||||||
|
"statement_id_prefix": ""
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA==",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_lambda_function.etl_post_processing",
|
||||||
|
"aws_lambda_layer_version.aws_wrangler",
|
||||||
|
"aws_s3_bucket.raw_bucket"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_s3_bucket",
|
||||||
|
"name": "athena_results",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"acceleration_status": "",
|
||||||
|
"acl": null,
|
||||||
|
"arn": "arn:aws:s3:::athena-results-613161579745-rc-s1201682",
|
||||||
|
"bucket": "athena-results-613161579745-rc-s1201682",
|
||||||
|
"bucket_domain_name": "athena-results-613161579745-rc-s1201682.s3.amazonaws.com",
|
||||||
|
"bucket_prefix": "",
|
||||||
|
"bucket_regional_domain_name": "athena-results-613161579745-rc-s1201682.s3.us-east-1.amazonaws.com",
|
||||||
|
"cors_rule": [],
|
||||||
|
"force_destroy": true,
|
||||||
|
"grant": [
|
||||||
|
{
|
||||||
|
"id": "c8f9aeae204fe9ab20afa7fb31e38920a3327db76890550c9eede7a1ef6c47f7",
|
||||||
|
"permissions": [
|
||||||
|
"FULL_CONTROL"
|
||||||
|
],
|
||||||
|
"type": "CanonicalUser",
|
||||||
|
"uri": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||||
|
"id": "athena-results-613161579745-rc-s1201682",
|
||||||
|
"lifecycle_rule": [],
|
||||||
|
"logging": [],
|
||||||
|
"object_lock_configuration": [],
|
||||||
|
"object_lock_enabled": false,
|
||||||
|
"policy": "",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"replication_configuration": [],
|
||||||
|
"request_payer": "BucketOwner",
|
||||||
|
"server_side_encryption_configuration": [
|
||||||
|
{
|
||||||
|
"rule": [
|
||||||
|
{
|
||||||
|
"apply_server_side_encryption_by_default": [
|
||||||
|
{
|
||||||
|
"kms_master_key_id": "",
|
||||||
|
"sse_algorithm": "AES256"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"bucket_key_enabled": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": {
|
||||||
|
"environment": "DEV",
|
||||||
|
"owner": "Rafal Charzewski",
|
||||||
|
"purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"environment": "DEV",
|
||||||
|
"owner": "Rafal Charzewski",
|
||||||
|
"purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"timeouts": null,
|
||||||
|
"versioning": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"mfa_delete": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"website": [],
|
||||||
|
"website_domain": null,
|
||||||
|
"website_endpoint": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_s3_bucket",
|
||||||
|
"name": "processed_bucket",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"acceleration_status": "",
|
||||||
|
"acl": null,
|
||||||
|
"arn": "arn:aws:s3:::datalake-processed-613161579745-rc-s1201682",
|
||||||
|
"bucket": "datalake-processed-613161579745-rc-s1201682",
|
||||||
|
"bucket_domain_name": "datalake-processed-613161579745-rc-s1201682.s3.amazonaws.com",
|
||||||
|
"bucket_prefix": "",
|
||||||
|
"bucket_regional_domain_name": "datalake-processed-613161579745-rc-s1201682.s3.us-east-1.amazonaws.com",
|
||||||
|
"cors_rule": [],
|
||||||
|
"force_destroy": true,
|
||||||
|
"grant": [
|
||||||
|
{
|
||||||
|
"id": "c8f9aeae204fe9ab20afa7fb31e38920a3327db76890550c9eede7a1ef6c47f7",
|
||||||
|
"permissions": [
|
||||||
|
"FULL_CONTROL"
|
||||||
|
],
|
||||||
|
"type": "CanonicalUser",
|
||||||
|
"uri": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||||
|
"id": "datalake-processed-613161579745-rc-s1201682",
|
||||||
|
"lifecycle_rule": [],
|
||||||
|
"logging": [],
|
||||||
|
"object_lock_configuration": [],
|
||||||
|
"object_lock_enabled": false,
|
||||||
|
"policy": "",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"replication_configuration": [],
|
||||||
|
"request_payer": "BucketOwner",
|
||||||
|
"server_side_encryption_configuration": [
|
||||||
|
{
|
||||||
|
"rule": [
|
||||||
|
{
|
||||||
|
"apply_server_side_encryption_by_default": [
|
||||||
|
{
|
||||||
|
"kms_master_key_id": "",
|
||||||
|
"sse_algorithm": "AES256"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"bucket_key_enabled": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": {
|
||||||
|
"Environment": "PROD",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"Environment": "PROD",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"timeouts": null,
|
||||||
|
"versioning": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"mfa_delete": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"website": [],
|
||||||
|
"website_domain": null,
|
||||||
|
"website_endpoint": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_s3_bucket",
|
||||||
|
"name": "raw_bucket",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"acceleration_status": "",
|
||||||
|
"acl": null,
|
||||||
|
"arn": "arn:aws:s3:::datalake-raw-613161579745-rc-s1201682",
|
||||||
|
"bucket": "datalake-raw-613161579745-rc-s1201682",
|
||||||
|
"bucket_domain_name": "datalake-raw-613161579745-rc-s1201682.s3.amazonaws.com",
|
||||||
|
"bucket_prefix": "",
|
||||||
|
"bucket_regional_domain_name": "datalake-raw-613161579745-rc-s1201682.s3.us-east-1.amazonaws.com",
|
||||||
|
"cors_rule": [],
|
||||||
|
"force_destroy": true,
|
||||||
|
"grant": [
|
||||||
|
{
|
||||||
|
"id": "c8f9aeae204fe9ab20afa7fb31e38920a3327db76890550c9eede7a1ef6c47f7",
|
||||||
|
"permissions": [
|
||||||
|
"FULL_CONTROL"
|
||||||
|
],
|
||||||
|
"type": "CanonicalUser",
|
||||||
|
"uri": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||||
|
"id": "datalake-raw-613161579745-rc-s1201682",
|
||||||
|
"lifecycle_rule": [],
|
||||||
|
"logging": [],
|
||||||
|
"object_lock_configuration": [],
|
||||||
|
"object_lock_enabled": false,
|
||||||
|
"policy": "",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"replication_configuration": [],
|
||||||
|
"request_payer": "BucketOwner",
|
||||||
|
"server_side_encryption_configuration": [
|
||||||
|
{
|
||||||
|
"rule": [
|
||||||
|
{
|
||||||
|
"apply_server_side_encryption_by_default": [
|
||||||
|
{
|
||||||
|
"kms_master_key_id": "",
|
||||||
|
"sse_algorithm": "AES256"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"bucket_key_enabled": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"timeouts": null,
|
||||||
|
"versioning": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"mfa_delete": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"website": [],
|
||||||
|
"website_domain": null,
|
||||||
|
"website_endpoint": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_s3_bucket_lifecycle_configuration",
|
||||||
|
"name": "athena_results_lifecycle",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"bucket": "athena-results-613161579745-rc-s1201682",
|
||||||
|
"expected_bucket_owner": "",
|
||||||
|
"id": "athena-results-613161579745-rc-s1201682",
|
||||||
|
"rule": [
|
||||||
|
{
|
||||||
|
"abort_incomplete_multipart_upload": [],
|
||||||
|
"expiration": [
|
||||||
|
{
|
||||||
|
"date": "",
|
||||||
|
"days": 1,
|
||||||
|
"expired_object_delete_marker": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"filter": [
|
||||||
|
{
|
||||||
|
"and": [],
|
||||||
|
"object_size_greater_than": "",
|
||||||
|
"object_size_less_than": "",
|
||||||
|
"prefix": "",
|
||||||
|
"tag": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": "standard-expiration",
|
||||||
|
"noncurrent_version_expiration": [],
|
||||||
|
"noncurrent_version_transition": [],
|
||||||
|
"prefix": "",
|
||||||
|
"status": "Enabled",
|
||||||
|
"transition": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"timeouts": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_s3_bucket.athena_results"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_s3_bucket_notification",
|
||||||
|
"name": "trigger_etl_lambda",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"bucket": "datalake-raw-613161579745-rc-s1201682",
|
||||||
|
"eventbridge": false,
|
||||||
|
"id": "datalake-raw-613161579745-rc-s1201682",
|
||||||
|
"lambda_function": [
|
||||||
|
{
|
||||||
|
"events": [
|
||||||
|
"s3:ObjectCreated:*"
|
||||||
|
],
|
||||||
|
"filter_prefix": "raw-zone/",
|
||||||
|
"filter_suffix": "",
|
||||||
|
"id": "tf-s3-lambda-20240619224612776100000001",
|
||||||
|
"lambda_function_arn": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"queue": [],
|
||||||
|
"topic": []
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA==",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_lambda_function.etl_post_processing",
|
||||||
|
"aws_lambda_layer_version.aws_wrangler",
|
||||||
|
"aws_lambda_permission.allow_bucket",
|
||||||
|
"aws_s3_bucket.raw_bucket"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"check_results": null
|
||||||
|
}
|
@ -1,5 +1,5 @@
|
|||||||
account_number=XXXXX
|
account_number=613161579745
|
||||||
student_initials="jk"
|
student_initials="rc"
|
||||||
student_full_name="Jakub Kasprzak"
|
student_full_name="Rafal Charzewski"
|
||||||
student_index_no = "12345"
|
student_index_no = "s1201682"
|
||||||
lab_role_arn = "arn:aws:iam::XXXXX:role/LabRole"
|
lab_role_arn = "arn:aws:iam::613161579745:role/LabRole"
|
Loading…
Reference in New Issue
Block a user