added new generator

This commit is contained in:
EC2 Default User 2024-05-09 20:39:17 +00:00
parent 8228ac3005
commit 272fff7769
3 changed files with 482 additions and 322 deletions

View File

@ -0,0 +1,160 @@
#!/usr/bin/env python3
import configparser
import argparse
import csv
import time
import logging
import sys
import json
import os
import boto3
logging.basicConfig(
level=logging.INFO,
format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
handlers=[logging.StreamHandler(sys.stdout)]
)
logger = logging.getLogger(__name__)
DEFAULT_DATA_FILE = 'crypto_trades_20201001.csv'
class KinesisProducer:
"""
Kinesis Producer
"""
def __init__(self, speed_per_sec):
self.client = boto3.client('kinesis')
self.max_retry_attempt = 5
def produce(self, events, data_stream):
"""
A simple wrapper for put record
:param event:
:param key:
:param data_stream:
:return:
"""
# adding a new line at the end to produce JSON lines
# (otherwise we would need to pre-process those records in Firehose
# invoking a Lambda to add those new lines).Every message is a dumped json with \n
#tran_id = event["trans_id"]
# payload = (json.dumps(event) + '\n').encode('utf-8')
attempt = 1
logger.info(events)
while attempt < self.max_retry_attempt:
try:
response = self.client.put_records(Records=events, StreamName=data_stream)
logger.info('Succesfully sended')
return response
except Exception as e:
logger.warning('Exception has occurred {}, retrying...'.format(e))
attempt += 1
time.sleep(attempt)
logger.error('Max attempt has been reached, rethrowing the last err')
raise
def prepare_event(event):
"""
Events from CSV have no dtypes, lets convert it to some more real values (int / decimals etc)
:param event:
:return:
"""
msg_key = event["symbol"]
msg_formatted = {
"transaction_ts": int(event["transaction_ts"]),
"symbol": event["symbol"],
"price": float(event["price"]),
"amount": float(event["amount"]),
"dollar_amount": float(event["dollar_amount"]),
"type": event["type"],
"trans_id": int(event["trans_id"]),
}
return msg_formatted, msg_key
def produce_data(kinesis_data_stream, messages_per_sec, input_file, single_run):
"""
Main method for producing
:param kinesis_data_stream: param from cmdline name of KDS
:param messages_per_sec: param from cmdline max speed per sec 1/mps
:return:
"""
kp = KinesisProducer(speed_per_sec=messages_per_sec)
with open(input_file) as csv_file:
events = []
reader = csv.DictReader(csv_file, delimiter=',')
all_rows = list(reader)
current_time = int(all_rows[0]["transaction_ts"])
replay_cnt = 1
events_cnt = 0
while True:
logger.info("start replaying for the {} time".format(replay_cnt))
for row in all_rows:
new_event_time = int(row["transaction_ts"])
time_delta = new_event_time - current_time
current_time = new_event_time
if time_delta > 0 and messages_per_sec > 0:
time.sleep(time_delta / messages_per_sec)
event, key = prepare_event(row)
# event_prepared = str(event)
data_for_event = {'Data':str(event), 'PartitionKey':key}
events.append(data_for_event)
events_cnt+=1
print(events_cnt)
if events_cnt == 10:
kp.produce(events, kinesis_data_stream)
events = []
events_cnt=0
time.sleep(1)
if single_run:
break
replay_cnt += 1
if __name__ == "__main__":
logger.info('Starting Simple Kinesis Producer (replaying stock data)')
parser = argparse.ArgumentParser()
parser.add_argument('-k', '--kinesis_ds', dest='kinesis_ds', required=True)
parser.add_argument('-i', '--input_file', dest='input_file', required=False)
parser.add_argument('-s', '--messages_per_sec', dest='mps', type=int, default=-1, required=False)
parser.add_argument('-r', '--single-run', dest='singel_run', action='store_true', required=False, default=False)
args, unknown = parser.parse_known_args()
config = configparser.ConfigParser()
kinesis_data_stream = args.kinesis_ds
messages_per_sec = int(args.mps)
single_run = args.singel_run if hasattr(args, 'singel_run') else False
if args.input_file:
input_file = args.input_file
else:
main_path = os.path.abspath(os.path.dirname(__file__))
input_file = os.path.join(main_path, DEFAULT_DATA_FILE)
produce_data(kinesis_data_stream, messages_per_sec, input_file, single_run)

View File

@ -1,9 +1,327 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.8.1", "terraform_version": "1.8.1",
"serial": 25, "serial": 43,
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
"outputs": {}, "outputs": {},
"resources": [], "resources": [
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680",
"catalog_id": "534534002841",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "534534002841:datalake_raw_534534002841_ab_1201680",
"location_uri": "",
"name": "datalake_raw_534534002841_ab_1201680",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_firehose_delivery_stream",
"name": "stock_delivery_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"destination": "extended_s3",
"destination_id": "destinationId-000000000001",
"elasticsearch_configuration": [],
"extended_s3_configuration": [
{
"bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"buffering_interval": 60,
"buffering_size": 1,
"cloudwatch_logging_options": [
{
"enabled": false,
"log_group_name": "",
"log_stream_name": ""
}
],
"compression_format": "UNCOMPRESSED",
"custom_time_zone": "UTC",
"data_format_conversion_configuration": [],
"dynamic_partitioning_configuration": [],
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"file_extension": "",
"kms_key_arn": "",
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"processing_configuration": [
{
"enabled": false,
"processors": []
}
],
"role_arn": "arn:aws:iam::534534002841:role/LabRole",
"s3_backup_configuration": [],
"s3_backup_mode": "Disabled"
}
],
"http_endpoint_configuration": [],
"id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"kinesis_source_configuration": [
{
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"role_arn": "arn:aws:iam::534534002841:role/LabRole"
}
],
"msk_source_configuration": [],
"name": "firehose-534534002841-ab-1201680",
"opensearch_configuration": [],
"opensearchserverless_configuration": [],
"redshift_configuration": [],
"server_side_encryption": [
{
"enabled": false,
"key_arn": "",
"key_type": "AWS_OWNED_CMK"
}
],
"snowflake_configuration": [],
"splunk_configuration": [],
"tags": null,
"tags_all": {},
"timeouts": null,
"version_id": "1"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
"dependencies": [
"aws_kinesis_stream.cryptostock_stream",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_stream",
"name": "cryptostock_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"encryption_type": "NONE",
"enforce_consumer_deletion": true,
"id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"kms_key_id": "",
"name": "cryptostock-534534002841-ab-1201680",
"retention_period": 24,
"shard_count": 1,
"shard_level_metrics": [
"IncomingBytes",
"IncomingRecords",
"OutgoingBytes",
"OutgoingRecords"
],
"stream_mode_details": [
{
"stream_mode": "PROVISIONED"
}
],
"tags": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "processed_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680",
"bucket": "datalake-processed-534534002841-ab-1201680",
"bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-processed-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "raw_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"bucket": "datalake-raw-534534002841-ab-1201680",
"bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-raw-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
}
],
"check_results": null "check_results": null
} }

View File

@ -1,327 +1,9 @@
{ {
"version": 4, "version": 4,
"terraform_version": "1.8.1", "terraform_version": "1.8.1",
"serial": 19, "serial": 37,
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
"outputs": {}, "outputs": {},
"resources": [ "resources": [],
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680",
"catalog_id": "534534002841",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "534534002841:datalake_raw_534534002841_ab_1201680",
"location_uri": "",
"name": "datalake_raw_534534002841_ab_1201680",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_firehose_delivery_stream",
"name": "stock_delivery_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"destination": "extended_s3",
"destination_id": "destinationId-000000000001",
"elasticsearch_configuration": [],
"extended_s3_configuration": [
{
"bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"buffering_interval": 60,
"buffering_size": 1,
"cloudwatch_logging_options": [
{
"enabled": false,
"log_group_name": "",
"log_stream_name": ""
}
],
"compression_format": "UNCOMPRESSED",
"custom_time_zone": "UTC",
"data_format_conversion_configuration": [],
"dynamic_partitioning_configuration": [],
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"file_extension": "",
"kms_key_arn": "",
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"processing_configuration": [
{
"enabled": false,
"processors": []
}
],
"role_arn": "arn:aws:iam::534534002841:role/LabRole",
"s3_backup_configuration": [],
"s3_backup_mode": "Disabled"
}
],
"http_endpoint_configuration": [],
"id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"kinesis_source_configuration": [
{
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"role_arn": "arn:aws:iam::534534002841:role/LabRole"
}
],
"msk_source_configuration": [],
"name": "firehose-534534002841-ab-1201680",
"opensearch_configuration": [],
"opensearchserverless_configuration": [],
"redshift_configuration": [],
"server_side_encryption": [
{
"enabled": false,
"key_arn": "",
"key_type": "AWS_OWNED_CMK"
}
],
"snowflake_configuration": [],
"splunk_configuration": [],
"tags": {},
"tags_all": {},
"timeouts": null,
"version_id": "1"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
"dependencies": [
"aws_kinesis_stream.cryptostock_stream",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_stream",
"name": "cryptostock_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"encryption_type": "NONE",
"enforce_consumer_deletion": true,
"id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"kms_key_id": "",
"name": "cryptostock-534534002841-ab-1201680",
"retention_period": 24,
"shard_count": 1,
"shard_level_metrics": [
"IncomingBytes",
"IncomingRecords",
"OutgoingBytes",
"OutgoingRecords"
],
"stream_mode_details": [
{
"stream_mode": "PROVISIONED"
}
],
"tags": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "processed_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680",
"bucket": "datalake-processed-534534002841-ab-1201680",
"bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-processed-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "raw_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"bucket": "datalake-raw-534534002841-ab-1201680",
"bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-raw-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
}
],
"check_results": null "check_results": null
} }