From 272fff77693da04aa87ee514c9ae60f700ff03f8 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 9 May 2024 20:39:17 +0000 Subject: [PATCH] added new generator --- labs/data_generator/generator_new.py | 160 ++++++++++++ labs/terraform/terraform.tfstate | 322 +++++++++++++++++++++++- labs/terraform/terraform.tfstate.backup | 322 +----------------------- 3 files changed, 482 insertions(+), 322 deletions(-) create mode 100644 labs/data_generator/generator_new.py diff --git a/labs/data_generator/generator_new.py b/labs/data_generator/generator_new.py new file mode 100644 index 0000000..5091fd6 --- /dev/null +++ b/labs/data_generator/generator_new.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +import configparser +import argparse + +import csv +import time +import logging +import sys +import json +import os + +import boto3 + +logging.basicConfig( + level=logging.INFO, + format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', + handlers=[logging.StreamHandler(sys.stdout)] +) +logger = logging.getLogger(__name__) + +DEFAULT_DATA_FILE = 'crypto_trades_20201001.csv' + + +class KinesisProducer: + """ + Kinesis Producer + """ + + def __init__(self, speed_per_sec): + + self.client = boto3.client('kinesis') + self.max_retry_attempt = 5 + + def produce(self, events, data_stream): + """ + A simple wrapper for put record + :param event: + :param key: + :param data_stream: + :return: + """ + + # adding a new line at the end to produce JSON lines + # (otherwise we would need to pre-process those records in Firehose + # invoking a Lambda to add those new lines).Every message is a dumped json with \n + + #tran_id = event["trans_id"] + # payload = (json.dumps(event) + '\n').encode('utf-8') + + attempt = 1 + logger.info(events) + while attempt < self.max_retry_attempt: + try: + response = self.client.put_records(Records=events, StreamName=data_stream) + logger.info('Succesfully sended') + + return response + + except Exception as e: + logger.warning('Exception has occurred {}, retrying...'.format(e)) + attempt += 1 + time.sleep(attempt) + + logger.error('Max attempt has been reached, rethrowing the last err') + raise + + +def prepare_event(event): + """ + Events from CSV have no dtypes, lets convert it to some more real values (int / decimals etc) + :param event: + :return: + """ + msg_key = event["symbol"] + + msg_formatted = { + "transaction_ts": int(event["transaction_ts"]), + "symbol": event["symbol"], + "price": float(event["price"]), + "amount": float(event["amount"]), + "dollar_amount": float(event["dollar_amount"]), + "type": event["type"], + "trans_id": int(event["trans_id"]), + } + + return msg_formatted, msg_key + + +def produce_data(kinesis_data_stream, messages_per_sec, input_file, single_run): + """ + Main method for producing + :param kinesis_data_stream: param from cmdline name of KDS + :param messages_per_sec: param from cmdline max speed per sec 1/mps + :return: + """ + kp = KinesisProducer(speed_per_sec=messages_per_sec) + + with open(input_file) as csv_file: + + events = [] + + reader = csv.DictReader(csv_file, delimiter=',') + all_rows = list(reader) + + current_time = int(all_rows[0]["transaction_ts"]) + + replay_cnt = 1 + events_cnt = 0 + + while True: + logger.info("start replaying for the {} time".format(replay_cnt)) + for row in all_rows: + + new_event_time = int(row["transaction_ts"]) + time_delta = new_event_time - current_time + current_time = new_event_time + + if time_delta > 0 and messages_per_sec > 0: + time.sleep(time_delta / messages_per_sec) + + event, key = prepare_event(row) + # event_prepared = str(event) + data_for_event = {'Data':str(event), 'PartitionKey':key} + events.append(data_for_event) + events_cnt+=1 + print(events_cnt) + if events_cnt == 10: + kp.produce(events, kinesis_data_stream) + events = [] + events_cnt=0 + time.sleep(1) + if single_run: + break + replay_cnt += 1 + + +if __name__ == "__main__": + logger.info('Starting Simple Kinesis Producer (replaying stock data)') + + parser = argparse.ArgumentParser() + parser.add_argument('-k', '--kinesis_ds', dest='kinesis_ds', required=True) + parser.add_argument('-i', '--input_file', dest='input_file', required=False) + parser.add_argument('-s', '--messages_per_sec', dest='mps', type=int, default=-1, required=False) + parser.add_argument('-r', '--single-run', dest='singel_run', action='store_true', required=False, default=False) + + args, unknown = parser.parse_known_args() + config = configparser.ConfigParser() + + kinesis_data_stream = args.kinesis_ds + messages_per_sec = int(args.mps) + + single_run = args.singel_run if hasattr(args, 'singel_run') else False + + if args.input_file: + input_file = args.input_file + else: + main_path = os.path.abspath(os.path.dirname(__file__)) + input_file = os.path.join(main_path, DEFAULT_DATA_FILE) + + produce_data(kinesis_data_stream, messages_per_sec, input_file, single_run) diff --git a/labs/terraform/terraform.tfstate b/labs/terraform/terraform.tfstate index 4274840..d153378 100644 --- a/labs/terraform/terraform.tfstate +++ b/labs/terraform/terraform.tfstate @@ -1,9 +1,327 @@ { "version": 4, "terraform_version": "1.8.1", - "serial": 25, + "serial": 43, "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", "outputs": {}, - "resources": [], + "resources": [ + { + "mode": "managed", + "type": "aws_glue_catalog_database", + "name": "datalake_db_raw_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680", + "catalog_id": "534534002841", + "create_table_default_permission": [ + { + "permissions": [ + "ALL" + ], + "principal": [ + { + "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" + } + ] + } + ], + "description": "", + "federated_database": [], + "id": "534534002841:datalake_raw_534534002841_ab_1201680", + "location_uri": "", + "name": "datalake_raw_534534002841_ab_1201680", + "parameters": null, + "tags": null, + "tags_all": {}, + "target_database": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_firehose_delivery_stream", + "name": "stock_delivery_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", + "destination": "extended_s3", + "destination_id": "destinationId-000000000001", + "elasticsearch_configuration": [], + "extended_s3_configuration": [ + { + "bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", + "buffering_interval": 60, + "buffering_size": 1, + "cloudwatch_logging_options": [ + { + "enabled": false, + "log_group_name": "", + "log_stream_name": "" + } + ], + "compression_format": "UNCOMPRESSED", + "custom_time_zone": "UTC", + "data_format_conversion_configuration": [], + "dynamic_partitioning_configuration": [], + "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "file_extension": "", + "kms_key_arn": "", + "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "processing_configuration": [ + { + "enabled": false, + "processors": [] + } + ], + "role_arn": "arn:aws:iam::534534002841:role/LabRole", + "s3_backup_configuration": [], + "s3_backup_mode": "Disabled" + } + ], + "http_endpoint_configuration": [], + "id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", + "kinesis_source_configuration": [ + { + "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "role_arn": "arn:aws:iam::534534002841:role/LabRole" + } + ], + "msk_source_configuration": [], + "name": "firehose-534534002841-ab-1201680", + "opensearch_configuration": [], + "opensearchserverless_configuration": [], + "redshift_configuration": [], + "server_side_encryption": [ + { + "enabled": false, + "key_arn": "", + "key_type": "AWS_OWNED_CMK" + } + ], + "snowflake_configuration": [], + "splunk_configuration": [], + "tags": null, + "tags_all": {}, + "timeouts": null, + "version_id": "1" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "aws_kinesis_stream.cryptostock_stream", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_stream", + "name": "cryptostock_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "encryption_type": "NONE", + "enforce_consumer_deletion": true, + "id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "kms_key_id": "", + "name": "cryptostock-534534002841-ab-1201680", + "retention_period": 24, + "shard_count": 1, + "shard_level_metrics": [ + "IncomingBytes", + "IncomingRecords", + "OutgoingBytes", + "OutgoingRecords" + ], + "stream_mode_details": [ + { + "stream_mode": "PROVISIONED" + } + ], + "tags": { + "Environment": "DEV", + "Owner": "Aron Boguszewski", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Owner": "Aron Boguszewski", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "processed_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680", + "bucket": "datalake-processed-534534002841-ab-1201680", + "bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-processed-534534002841-ab-1201680", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "raw_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", + "bucket": "datalake-raw-534534002841-ab-1201680", + "bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-raw-534534002841-ab-1201680", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], "check_results": null } diff --git a/labs/terraform/terraform.tfstate.backup b/labs/terraform/terraform.tfstate.backup index 9049295..3e3b299 100644 --- a/labs/terraform/terraform.tfstate.backup +++ b/labs/terraform/terraform.tfstate.backup @@ -1,327 +1,9 @@ { "version": 4, "terraform_version": "1.8.1", - "serial": 19, + "serial": 37, "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", "outputs": {}, - "resources": [ - { - "mode": "managed", - "type": "aws_glue_catalog_database", - "name": "datalake_db_raw_zone", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680", - "catalog_id": "534534002841", - "create_table_default_permission": [ - { - "permissions": [ - "ALL" - ], - "principal": [ - { - "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" - } - ] - } - ], - "description": "", - "federated_database": [], - "id": "534534002841:datalake_raw_534534002841_ab_1201680", - "location_uri": "", - "name": "datalake_raw_534534002841_ab_1201680", - "parameters": null, - "tags": null, - "tags_all": {}, - "target_database": [] - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "mode": "managed", - "type": "aws_kinesis_firehose_delivery_stream", - "name": "stock_delivery_stream", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", - "destination": "extended_s3", - "destination_id": "destinationId-000000000001", - "elasticsearch_configuration": [], - "extended_s3_configuration": [ - { - "bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", - "buffering_interval": 60, - "buffering_size": 1, - "cloudwatch_logging_options": [ - { - "enabled": false, - "log_group_name": "", - "log_stream_name": "" - } - ], - "compression_format": "UNCOMPRESSED", - "custom_time_zone": "UTC", - "data_format_conversion_configuration": [], - "dynamic_partitioning_configuration": [], - "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", - "file_extension": "", - "kms_key_arn": "", - "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", - "processing_configuration": [ - { - "enabled": false, - "processors": [] - } - ], - "role_arn": "arn:aws:iam::534534002841:role/LabRole", - "s3_backup_configuration": [], - "s3_backup_mode": "Disabled" - } - ], - "http_endpoint_configuration": [], - "id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", - "kinesis_source_configuration": [ - { - "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", - "role_arn": "arn:aws:iam::534534002841:role/LabRole" - } - ], - "msk_source_configuration": [], - "name": "firehose-534534002841-ab-1201680", - "opensearch_configuration": [], - "opensearchserverless_configuration": [], - "redshift_configuration": [], - "server_side_encryption": [ - { - "enabled": false, - "key_arn": "", - "key_type": "AWS_OWNED_CMK" - } - ], - "snowflake_configuration": [], - "splunk_configuration": [], - "tags": {}, - "tags_all": {}, - "timeouts": null, - "version_id": "1" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", - "dependencies": [ - "aws_kinesis_stream.cryptostock_stream", - "aws_s3_bucket.raw_bucket" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_kinesis_stream", - "name": "cryptostock_stream", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", - "encryption_type": "NONE", - "enforce_consumer_deletion": true, - "id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", - "kms_key_id": "", - "name": "cryptostock-534534002841-ab-1201680", - "retention_period": 24, - "shard_count": 1, - "shard_level_metrics": [ - "IncomingBytes", - "IncomingRecords", - "OutgoingBytes", - "OutgoingRecords" - ], - "stream_mode_details": [ - { - "stream_mode": "PROVISIONED" - } - ], - "tags": { - "Environment": "DEV", - "Owner": "Aron Boguszewski", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Owner": "Aron Boguszewski", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket", - "name": "processed_bucket", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acceleration_status": "", - "acl": null, - "arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680", - "bucket": "datalake-processed-534534002841-ab-1201680", - "bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com", - "bucket_prefix": "", - "bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", - "cors_rule": [], - "force_destroy": true, - "grant": [ - { - "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", - "permissions": [ - "FULL_CONTROL" - ], - "type": "CanonicalUser", - "uri": "" - } - ], - "hosted_zone_id": "Z3AQBSTGFYJSTF", - "id": "datalake-processed-534534002841-ab-1201680", - "lifecycle_rule": [], - "logging": [], - "object_lock_configuration": [], - "object_lock_enabled": false, - "policy": "", - "region": "us-east-1", - "replication_configuration": [], - "request_payer": "BucketOwner", - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": "", - "sse_algorithm": "AES256" - } - ], - "bucket_key_enabled": false - } - ] - } - ], - "tags": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null, - "versioning": [ - { - "enabled": false, - "mfa_delete": false - } - ], - "website": [], - "website_domain": null, - "website_endpoint": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket", - "name": "raw_bucket", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acceleration_status": "", - "acl": null, - "arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", - "bucket": "datalake-raw-534534002841-ab-1201680", - "bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com", - "bucket_prefix": "", - "bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", - "cors_rule": [], - "force_destroy": true, - "grant": [ - { - "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", - "permissions": [ - "FULL_CONTROL" - ], - "type": "CanonicalUser", - "uri": "" - } - ], - "hosted_zone_id": "Z3AQBSTGFYJSTF", - "id": "datalake-raw-534534002841-ab-1201680", - "lifecycle_rule": [], - "logging": [], - "object_lock_configuration": [], - "object_lock_enabled": false, - "policy": "", - "region": "us-east-1", - "replication_configuration": [], - "request_payer": "BucketOwner", - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": "", - "sse_algorithm": "AES256" - } - ], - "bucket_key_enabled": false - } - ] - } - ], - "tags": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null, - "versioning": [ - { - "enabled": false, - "mfa_delete": false - } - ], - "website": [], - "website_domain": null, - "website_endpoint": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - } - ], + "resources": [], "check_results": null }