Dodano nowe pliki i zmiany w kodzie Lambda

This commit is contained in:
EC2 Default User 2024-05-30 09:28:22 +00:00
parent b6145320e9
commit bc1f2ad333
4 changed files with 880 additions and 902 deletions

View File

@ -9,16 +9,19 @@ def etl_function(event, context):
processed_zone_prefix = "processed-zone" processed_zone_prefix = "processed-zone"
record = event["Records"][0] record = event["Records"][0]
bucket = client_ssm.get_parameter(Name = 's3_processed_bucket_name')['Parameter']['Value'] raw_bucket = record["s3"]["bucket"]["name"]
key = urllib.parse.unquote(record["s3"]["object"]["key"]) key = urllib.parse.unquote(record["s3"]["object"]["key"])
event_prefix = key.split('/')[1] event_prefix = key.split('/')[1]
full_src_path = 's3://{bucket}/{key}'.format(bucket=bucket, key=key) full_src_path = f's3://{raw_bucket}/{key}'
# Pobierz nazwę bucketu przetworzonych danych z SSM
processed_bucket = client_ssm.get_parameter(Name='s3_processed_bucket_name')['Parameter']['Value']
print(f'Processing key = {full_src_path}') print(f'Processing key = {full_src_path}')
df = wr.s3.read_json(path=full_src_path, lines=True) df = wr.s3.read_json(path=full_src_path, lines=True)
filename = key.split('/')[-1][-36:] filename = key.split('/')[-1][-36:]
dest_prefix = f"s3://{bucket}/{processed_zone_prefix}/{event_prefix}" dest_prefix = f"s3://{processed_bucket}/{processed_zone_prefix}/{event_prefix}"
df['transaction_date'] = pd.to_datetime(df['transaction_ts'], unit='s') df['transaction_date'] = pd.to_datetime(df['transaction_ts'], unit='s')
df['year'] = df['transaction_date'].dt.year df['year'] = df['transaction_date'].dt.year
@ -46,11 +49,4 @@ def etl_function(event, context):
'key': key, 'key': key,
'statusCode': 200, 'statusCode': 200,
'new_keys': new_keys 'new_keys': new_keys
} }
if __name__ == "__main__":
event = ""
context = ""
response = etl_function(event, context)

Binary file not shown.

View File

@ -1,9 +1,878 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.8.1", "terraform_version": "1.8.1",
"serial": 289, "serial": 340,
"lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff", "lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff",
"outputs": {}, "outputs": {},
"resources": [], "resources": [
{
"mode": "managed",
"type": "aws_athena_workgroup",
"name": "athena_workgroup",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:athena:us-east-1:205216182560:workgroup/development",
"configuration": [
{
"bytes_scanned_cutoff_per_query": 0,
"enforce_workgroup_configuration": true,
"engine_version": [
{
"effective_engine_version": "Athena engine version 3",
"selected_engine_version": "AUTO"
}
],
"execution_role": "",
"publish_cloudwatch_metrics_enabled": true,
"requester_pays_enabled": false,
"result_configuration": [
{
"acl_configuration": [],
"encryption_configuration": [],
"expected_bucket_owner": "",
"output_location": "s3://athena-results-205216182560-agb-s1201687/output/"
}
]
}
],
"description": "",
"force_destroy": true,
"id": "development",
"name": "development",
"state": "ENABLED",
"tags": null,
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_s3_bucket.athena_results"
]
}
]
},
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_processed_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_processed_205216182560_agb_s1201687",
"catalog_id": "205216182560",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "205216182560:datalake_processed_205216182560_agb_s1201687",
"location_uri": "",
"name": "datalake_processed_205216182560_agb_s1201687",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_raw_205216182560_agb_s1201687",
"catalog_id": "205216182560",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "205216182560:datalake_raw_205216182560_agb_s1201687",
"location_uri": "",
"name": "datalake_raw_205216182560_agb_s1201687",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_glue_crawler",
"name": "glue_crawler_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:205216182560:crawler/gc-raw-205216182560-agb-s1201687",
"catalog_target": [],
"classifiers": null,
"configuration": "",
"database_name": "datalake_raw_205216182560_agb_s1201687",
"delta_target": [],
"description": "",
"dynamodb_target": [],
"hudi_target": [],
"iceberg_target": [],
"id": "gc-raw-205216182560-agb-s1201687",
"jdbc_target": [],
"lake_formation_configuration": [
{
"account_id": "",
"use_lake_formation_credentials": false
}
],
"lineage_configuration": [
{
"crawler_lineage_settings": "DISABLE"
}
],
"mongodb_target": [],
"name": "gc-raw-205216182560-agb-s1201687",
"recrawl_policy": [
{
"recrawl_behavior": "CRAWL_EVERYTHING"
}
],
"role": "LabRole",
"s3_target": [
{
"connection_name": "",
"dlq_event_queue_arn": "",
"event_queue_arn": "",
"exclusions": null,
"path": "s3://datalake-raw-205216182560-agb-s1201687/raw-zone/stockdata/",
"sample_size": 0
}
],
"schedule": "",
"schema_change_policy": [
{
"delete_behavior": "DEPRECATE_IN_DATABASE",
"update_behavior": "UPDATE_IN_DATABASE"
}
],
"security_configuration": "",
"table_prefix": "crawler_",
"tags": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_glue_catalog_database.datalake_db_raw_zone",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_iam_policy",
"name": "lambda_policy",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:iam::205216182560:policy/lambda_policy",
"attachment_count": 0,
"description": "",
"id": "arn:aws:iam::205216182560:policy/lambda_policy",
"name": "lambda_policy",
"name_prefix": "",
"path": "/",
"policy": "{\"Statement\":[{\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::datalake-raw-205216182560-agb-s1201687/*\",\"arn:aws:s3:::datalake-processed-205216182560-agb-s1201687/*\"]},{\"Action\":\"ssm:GetParameter\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name\"}],\"Version\":\"2012-10-17\"}",
"policy_id": "ANPAS7R6WOEQMJ2ODN7S5",
"tags": null,
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_s3_bucket.processed_bucket",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_firehose_delivery_stream",
"name": "stock_delivery_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
"destination": "extended_s3",
"destination_id": "destinationId-000000000001",
"elasticsearch_configuration": [],
"extended_s3_configuration": [
{
"bucket_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
"buffering_interval": 60,
"buffering_size": 1,
"cloudwatch_logging_options": [
{
"enabled": false,
"log_group_name": "",
"log_stream_name": ""
}
],
"compression_format": "UNCOMPRESSED",
"custom_time_zone": "UTC",
"data_format_conversion_configuration": [],
"dynamic_partitioning_configuration": [],
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"file_extension": "",
"kms_key_arn": "",
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"processing_configuration": [
{
"enabled": false,
"processors": []
}
],
"role_arn": "arn:aws:iam::205216182560:role/LabRole",
"s3_backup_configuration": [],
"s3_backup_mode": "Disabled"
}
],
"http_endpoint_configuration": [],
"id": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
"kinesis_source_configuration": [
{
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
"role_arn": "arn:aws:iam::205216182560:role/LabRole"
}
],
"msk_source_configuration": [],
"name": "firehose-205216182560-agb-s1201687",
"opensearch_configuration": [],
"opensearchserverless_configuration": [],
"redshift_configuration": [],
"server_side_encryption": [
{
"enabled": false,
"key_arn": "",
"key_type": "AWS_OWNED_CMK"
}
],
"snowflake_configuration": [],
"splunk_configuration": [],
"tags": null,
"tags_all": {},
"timeouts": null,
"version_id": "1"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
"dependencies": [
"aws_kinesis_stream.cryptostock_stream",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_stream",
"name": "cryptostock_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
"encryption_type": "NONE",
"enforce_consumer_deletion": true,
"id": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
"kms_key_id": "",
"name": "cryptostock-205216182560-agb-s1201687",
"retention_period": 24,
"shard_count": 1,
"shard_level_metrics": [
"IncomingBytes",
"IncomingRecords",
"OutgoingBytes",
"OutgoingRecords"
],
"stream_mode_details": [
{
"stream_mode": "PROVISIONED"
}
],
"tags": {
"Environment": "DEV",
"Owner": "Agnieszka Gąbka-Buszek",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Owner": "Agnieszka Gąbka-Buszek",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_lambda_function",
"name": "etl_post_processing",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"architectures": [
"x86_64"
],
"arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
"code_signing_config_arn": "",
"dead_letter_config": [],
"description": "",
"environment": [],
"ephemeral_storage": [
{
"size": 512
}
],
"file_system_config": [],
"filename": "../lambda/lambda_definition.zip",
"function_name": "etl-post-processing-205216182560-agb-s1201687",
"handler": "lambda_definition.etl_function",
"id": "etl-post-processing-205216182560-agb-s1201687",
"image_config": [],
"image_uri": "",
"invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687/invocations",
"kms_key_arn": "",
"last_modified": "2024-05-30T09:19:35.178+0000",
"layers": [
"arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10"
],
"logging_config": [
{
"application_log_level": "",
"log_format": "Text",
"log_group": "/aws/lambda/etl-post-processing-205216182560-agb-s1201687",
"system_log_level": ""
}
],
"memory_size": 512,
"package_type": "Zip",
"publish": false,
"qualified_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST",
"qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST/invocations",
"replace_security_groups_on_destroy": null,
"replacement_security_group_ids": null,
"reserved_concurrent_executions": -1,
"role": "arn:aws:iam::205216182560:role/LabRole",
"runtime": "python3.8",
"s3_bucket": null,
"s3_key": null,
"s3_object_version": null,
"signing_job_arn": "",
"signing_profile_version_arn": "",
"skip_destroy": false,
"snap_start": [],
"source_code_hash": "rlA5g+hXywn0cpXGqkL9RY3YPioioHFCQQ8jp5Ssjvs=",
"source_code_size": 985,
"tags": null,
"tags_all": {},
"timeout": 300,
"timeouts": null,
"tracing_config": [
{
"mode": "PassThrough"
}
],
"version": "$LATEST",
"vpc_config": []
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19",
"dependencies": [
"aws_lambda_layer_version.aws_wrangler"
]
}
]
},
{
"mode": "managed",
"type": "aws_lambda_layer_version",
"name": "aws_wrangler",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10",
"compatible_architectures": null,
"compatible_runtimes": [
"python3.8"
],
"created_date": "2024-05-30T09:19:34.959+0000",
"description": "",
"filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip",
"id": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10",
"layer_arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687",
"layer_name": "aws_wrangler_205216182560_agb_s1201687",
"license_info": "",
"s3_bucket": null,
"s3_key": null,
"s3_object_version": null,
"signing_job_arn": "",
"signing_profile_version_arn": "",
"skip_destroy": false,
"source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=",
"source_code_size": 43879070,
"version": "10"
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_lambda_permission",
"name": "allow_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"action": "lambda:InvokeFunction",
"event_source_token": null,
"function_name": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
"function_url_auth_type": null,
"id": "AllowExecutionFromS3Bucket",
"principal": "s3.amazonaws.com",
"principal_org_id": null,
"qualifier": "",
"source_account": null,
"source_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
"statement_id": "AllowExecutionFromS3Bucket",
"statement_id_prefix": ""
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_lambda_function.etl_post_processing",
"aws_lambda_layer_version.aws_wrangler",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "athena_results",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::athena-results-205216182560-agb-s1201687",
"bucket": "athena-results-205216182560-agb-s1201687",
"bucket_domain_name": "athena-results-205216182560-agb-s1201687.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "athena-results-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "athena-results-205216182560-agb-s1201687",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "processed_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-processed-205216182560-agb-s1201687",
"bucket": "datalake-processed-205216182560-agb-s1201687",
"bucket_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-processed-205216182560-agb-s1201687",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "raw_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
"bucket": "datalake-raw-205216182560-agb-s1201687",
"bucket_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-raw-205216182560-agb-s1201687",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket_lifecycle_configuration",
"name": "athena_results_lifecycle",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"bucket": "athena-results-205216182560-agb-s1201687",
"expected_bucket_owner": "",
"id": "athena-results-205216182560-agb-s1201687",
"rule": [
{
"abort_incomplete_multipart_upload": [],
"expiration": [
{
"date": "",
"days": 1,
"expired_object_delete_marker": false
}
],
"filter": [
{
"and": [],
"object_size_greater_than": "",
"object_size_less_than": "",
"prefix": "",
"tag": []
}
],
"id": "standard-expiration",
"noncurrent_version_expiration": [],
"noncurrent_version_transition": [],
"prefix": "",
"status": "Enabled",
"transition": []
}
],
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=",
"dependencies": [
"aws_s3_bucket.athena_results"
]
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket_notification",
"name": "trigger_etl_lambda",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"bucket": "datalake-raw-205216182560-agb-s1201687",
"eventbridge": false,
"id": "datalake-raw-205216182560-agb-s1201687",
"lambda_function": [
{
"events": [
"s3:ObjectCreated:*"
],
"filter_prefix": "raw-zone/",
"filter_suffix": "",
"id": "tf-s3-lambda-20240530091945493300000001",
"lambda_function_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687"
}
],
"queue": [],
"topic": []
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_lambda_function.etl_post_processing",
"aws_lambda_layer_version.aws_wrangler",
"aws_lambda_permission.allow_bucket",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_ssm_parameter",
"name": "s3_processed",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"allowed_pattern": "",
"arn": "arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name",
"data_type": "text",
"description": "",
"id": "s3_processed_bucket_name",
"insecure_value": null,
"key_id": "",
"name": "s3_processed_bucket_name",
"overwrite": null,
"tags": null,
"tags_all": {},
"tier": "Standard",
"type": "String",
"value": "datalake-processed-205216182560-agb-s1201687",
"version": 1
},
"sensitive_attributes": [
[
{
"type": "get_attr",
"value": "value"
}
]
],
"private": "bnVsbA==",
"dependencies": [
"aws_s3_bucket.processed_bucket"
]
}
]
}
],
"check_results": null "check_results": null
} }

View File

@ -1,896 +1,9 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.8.1", "terraform_version": "1.8.1",
"serial": 272, "serial": 323,
"lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff", "lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff",
"outputs": {}, "outputs": {},
"resources": [ "resources": [],
{
"mode": "managed",
"type": "aws_athena_workgroup",
"name": "athena_workgroup",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:athena:us-east-1:205216182560:workgroup/development",
"configuration": [
{
"bytes_scanned_cutoff_per_query": 0,
"enforce_workgroup_configuration": true,
"engine_version": [
{
"effective_engine_version": "Athena engine version 3",
"selected_engine_version": "AUTO"
}
],
"execution_role": "",
"publish_cloudwatch_metrics_enabled": true,
"requester_pays_enabled": false,
"result_configuration": [
{
"acl_configuration": [],
"encryption_configuration": [],
"expected_bucket_owner": "",
"output_location": "s3://athena-results-205216182560-agb-s1201687/output/"
}
]
}
],
"description": "",
"force_destroy": true,
"id": "development",
"name": "development",
"state": "ENABLED",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_s3_bucket.athena_results"
]
}
]
},
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_processed_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_processed_205216182560_agb_s1201687",
"catalog_id": "205216182560",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "205216182560:datalake_processed_205216182560_agb_s1201687",
"location_uri": "",
"name": "datalake_processed_205216182560_agb_s1201687",
"parameters": {},
"tags": {},
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_raw_205216182560_agb_s1201687",
"catalog_id": "205216182560",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "205216182560:datalake_raw_205216182560_agb_s1201687",
"location_uri": "",
"name": "datalake_raw_205216182560_agb_s1201687",
"parameters": {},
"tags": {},
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_glue_crawler",
"name": "glue_crawler_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:205216182560:crawler/gc-raw-205216182560-agb-s1201687",
"catalog_target": [],
"classifiers": [],
"configuration": "",
"database_name": "datalake_raw_205216182560_agb_s1201687",
"delta_target": [],
"description": "",
"dynamodb_target": [],
"hudi_target": [],
"iceberg_target": [],
"id": "gc-raw-205216182560-agb-s1201687",
"jdbc_target": [],
"lake_formation_configuration": [
{
"account_id": "",
"use_lake_formation_credentials": false
}
],
"lineage_configuration": [
{
"crawler_lineage_settings": "DISABLE"
}
],
"mongodb_target": [],
"name": "gc-raw-205216182560-agb-s1201687",
"recrawl_policy": [
{
"recrawl_behavior": "CRAWL_EVERYTHING"
}
],
"role": "LabRole",
"s3_target": [
{
"connection_name": "",
"dlq_event_queue_arn": "",
"event_queue_arn": "",
"exclusions": [],
"path": "s3://datalake-raw-205216182560-agb-s1201687/raw-zone/stockdata/",
"sample_size": 0
}
],
"schedule": "",
"schema_change_policy": [
{
"delete_behavior": "DEPRECATE_IN_DATABASE",
"update_behavior": "UPDATE_IN_DATABASE"
}
],
"security_configuration": "",
"table_prefix": "crawler_",
"tags": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_glue_catalog_database.datalake_db_raw_zone",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_iam_policy",
"name": "lambda_policy",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:iam::205216182560:policy/lambda_policy",
"attachment_count": 0,
"description": "",
"id": "arn:aws:iam::205216182560:policy/lambda_policy",
"name": "lambda_policy",
"name_prefix": "",
"path": "/",
"policy": "{\"Statement\":[{\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::datalake-raw-205216182560-agb-s1201687/*\",\"arn:aws:s3:::datalake-processed-205216182560-agb-s1201687/*\"]},{\"Action\":\"ssm:GetParameter\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name\"}],\"Version\":\"2012-10-17\"}",
"policy_id": "ANPAS7R6WOEQGHVSNQS6X",
"tags": {},
"tags_all": {}
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_s3_bucket.processed_bucket",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_firehose_delivery_stream",
"name": "stock_delivery_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
"destination": "extended_s3",
"destination_id": "destinationId-000000000001",
"elasticsearch_configuration": [],
"extended_s3_configuration": [
{
"bucket_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
"buffering_interval": 60,
"buffering_size": 1,
"cloudwatch_logging_options": [
{
"enabled": false,
"log_group_name": "",
"log_stream_name": ""
}
],
"compression_format": "UNCOMPRESSED",
"custom_time_zone": "UTC",
"data_format_conversion_configuration": [],
"dynamic_partitioning_configuration": [],
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"file_extension": "",
"kms_key_arn": "",
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"processing_configuration": [
{
"enabled": false,
"processors": []
}
],
"role_arn": "arn:aws:iam::205216182560:role/LabRole",
"s3_backup_configuration": [],
"s3_backup_mode": "Disabled"
}
],
"http_endpoint_configuration": [],
"id": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
"kinesis_source_configuration": [
{
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
"role_arn": "arn:aws:iam::205216182560:role/LabRole"
}
],
"msk_source_configuration": [],
"name": "firehose-205216182560-agb-s1201687",
"opensearch_configuration": [],
"opensearchserverless_configuration": [],
"redshift_configuration": [],
"server_side_encryption": [
{
"enabled": false,
"key_arn": "",
"key_type": "AWS_OWNED_CMK"
}
],
"snowflake_configuration": [],
"splunk_configuration": [],
"tags": {},
"tags_all": {},
"timeouts": null,
"version_id": "1"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
"dependencies": [
"aws_kinesis_stream.cryptostock_stream",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_stream",
"name": "cryptostock_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
"encryption_type": "NONE",
"enforce_consumer_deletion": true,
"id": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
"kms_key_id": "",
"name": "cryptostock-205216182560-agb-s1201687",
"retention_period": 24,
"shard_count": 1,
"shard_level_metrics": [
"IncomingBytes",
"IncomingRecords",
"OutgoingBytes",
"OutgoingRecords"
],
"stream_mode_details": [
{
"stream_mode": "PROVISIONED"
}
],
"tags": {
"Environment": "DEV",
"Owner": "Agnieszka Gąbka-Buszek",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Owner": "Agnieszka Gąbka-Buszek",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_lambda_function",
"name": "etl_post_processing",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"architectures": [
"x86_64"
],
"arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
"code_signing_config_arn": "",
"dead_letter_config": [],
"description": "",
"environment": [],
"ephemeral_storage": [
{
"size": 512
}
],
"file_system_config": [],
"filename": "../lambda/lambda_definition.zip",
"function_name": "etl-post-processing-205216182560-agb-s1201687",
"handler": "lambda_definition.etl_function",
"id": "etl-post-processing-205216182560-agb-s1201687",
"image_config": [],
"image_uri": "",
"invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687/invocations",
"kms_key_arn": "",
"last_modified": "2024-05-27T10:23:37.000+0000",
"layers": [
"arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7"
],
"logging_config": [
{
"application_log_level": "",
"log_format": "Text",
"log_group": "/aws/lambda/etl-post-processing-205216182560-agb-s1201687",
"system_log_level": ""
}
],
"memory_size": 512,
"package_type": "Zip",
"publish": false,
"qualified_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST",
"qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST/invocations",
"replace_security_groups_on_destroy": null,
"replacement_security_group_ids": null,
"reserved_concurrent_executions": -1,
"role": "arn:aws:iam::205216182560:role/LabRole",
"runtime": "python3.8",
"s3_bucket": null,
"s3_key": null,
"s3_object_version": null,
"signing_job_arn": "",
"signing_profile_version_arn": "",
"skip_destroy": false,
"snap_start": [],
"source_code_hash": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=",
"source_code_size": 884,
"tags": {},
"tags_all": {},
"timeout": 300,
"timeouts": null,
"tracing_config": [
{
"mode": "PassThrough"
}
],
"version": "$LATEST",
"vpc_config": []
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19",
"dependencies": [
"aws_lambda_layer_version.aws_wrangler"
]
}
]
},
{
"mode": "managed",
"type": "aws_lambda_layer_version",
"name": "aws_wrangler",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7",
"compatible_architectures": [],
"compatible_runtimes": [
"python3.8"
],
"created_date": "2024-05-27T08:03:57.293+0000",
"description": "",
"filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip",
"id": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7",
"layer_arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687",
"layer_name": "aws_wrangler_205216182560_agb_s1201687",
"license_info": "",
"s3_bucket": null,
"s3_key": null,
"s3_object_version": null,
"signing_job_arn": "",
"signing_profile_version_arn": "",
"skip_destroy": false,
"source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=",
"source_code_size": 43879070,
"version": "7"
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_lambda_permission",
"name": "allow_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"action": "lambda:InvokeFunction",
"event_source_token": null,
"function_name": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
"function_url_auth_type": null,
"id": "AllowExecutionFromS3Bucket",
"principal": "s3.amazonaws.com",
"principal_org_id": null,
"qualifier": "",
"source_account": null,
"source_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
"statement_id": "AllowExecutionFromS3Bucket",
"statement_id_prefix": ""
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_lambda_function.etl_post_processing",
"aws_lambda_layer_version.aws_wrangler",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "athena_results",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::athena-results-205216182560-agb-s1201687",
"bucket": "athena-results-205216182560-agb-s1201687",
"bucket_domain_name": "athena-results-205216182560-agb-s1201687.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "athena-results-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "athena-results-205216182560-agb-s1201687",
"lifecycle_rule": [
{
"abort_incomplete_multipart_upload_days": 0,
"enabled": true,
"expiration": [
{
"date": "",
"days": 1,
"expired_object_delete_marker": false
}
],
"id": "standard-expiration",
"noncurrent_version_expiration": [],
"noncurrent_version_transition": [],
"prefix": "",
"tags": {},
"transition": []
}
],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"environment": "DEV",
"owner": "Agnieszka Gąbka-Buszek",
"purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "processed_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-processed-205216182560-agb-s1201687",
"bucket": "datalake-processed-205216182560-agb-s1201687",
"bucket_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-processed-205216182560-agb-s1201687",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "raw_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
"bucket": "datalake-raw-205216182560-agb-s1201687",
"bucket_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-raw-205216182560-agb-s1201687",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket_lifecycle_configuration",
"name": "athena_results_lifecycle",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"bucket": "athena-results-205216182560-agb-s1201687",
"expected_bucket_owner": "",
"id": "athena-results-205216182560-agb-s1201687",
"rule": [
{
"abort_incomplete_multipart_upload": [],
"expiration": [
{
"date": "",
"days": 1,
"expired_object_delete_marker": false
}
],
"filter": [
{
"and": [],
"object_size_greater_than": "",
"object_size_less_than": "",
"prefix": "",
"tag": []
}
],
"id": "standard-expiration",
"noncurrent_version_expiration": [],
"noncurrent_version_transition": [],
"prefix": "",
"status": "Enabled",
"transition": []
}
],
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=",
"dependencies": [
"aws_s3_bucket.athena_results"
]
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket_notification",
"name": "trigger_etl_lambda",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"bucket": "datalake-raw-205216182560-agb-s1201687",
"eventbridge": false,
"id": "datalake-raw-205216182560-agb-s1201687",
"lambda_function": [
{
"events": [
"s3:ObjectCreated:*"
],
"filter_prefix": "raw-zone/",
"filter_suffix": "",
"id": "tf-s3-lambda-20240527080407879000000001",
"lambda_function_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687"
}
],
"queue": [],
"topic": []
},
"sensitive_attributes": [],
"private": "bnVsbA==",
"dependencies": [
"aws_lambda_function.etl_post_processing",
"aws_lambda_layer_version.aws_wrangler",
"aws_lambda_permission.allow_bucket",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_ssm_parameter",
"name": "s3_processed",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"allowed_pattern": "",
"arn": "arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name",
"data_type": "text",
"description": "",
"id": "s3_processed_bucket_name",
"insecure_value": null,
"key_id": "",
"name": "s3_processed_bucket_name",
"overwrite": null,
"tags": {},
"tags_all": {},
"tier": "Standard",
"type": "String",
"value": "datalake-processed-205216182560-agb-s1201687",
"version": 1
},
"sensitive_attributes": [
[
{
"type": "get_attr",
"value": "value"
}
]
],
"private": "bnVsbA==",
"dependencies": [
"aws_s3_bucket.processed_bucket"
]
}
]
}
],
"check_results": null "check_results": null
} }