added new generator
This commit is contained in:
EC2 Default User
parent
8228ac3005
commit
272fff7769
160
labs/data_generator/generator_new.py
Normal file
160
labs/data_generator/generator_new.py
Normal file
@ -0,0 +1,160 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import configparser
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
|
||||||
|
handlers=[logging.StreamHandler(sys.stdout)]
|
||||||
|
)
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_DATA_FILE = 'crypto_trades_20201001.csv'
|
||||||
|
|
||||||
|
|
||||||
|
class KinesisProducer:
|
||||||
|
"""
|
||||||
|
Kinesis Producer
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, speed_per_sec):
|
||||||
|
|
||||||
|
self.client = boto3.client('kinesis')
|
||||||
|
self.max_retry_attempt = 5
|
||||||
|
|
||||||
|
def produce(self, events, data_stream):
|
||||||
|
"""
|
||||||
|
A simple wrapper for put record
|
||||||
|
:param event:
|
||||||
|
:param key:
|
||||||
|
:param data_stream:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
|
||||||
|
# adding a new line at the end to produce JSON lines
|
||||||
|
# (otherwise we would need to pre-process those records in Firehose
|
||||||
|
# invoking a Lambda to add those new lines).Every message is a dumped json with \n
|
||||||
|
|
||||||
|
#tran_id = event["trans_id"]
|
||||||
|
# payload = (json.dumps(event) + '\n').encode('utf-8')
|
||||||
|
|
||||||
|
attempt = 1
|
||||||
|
logger.info(events)
|
||||||
|
while attempt < self.max_retry_attempt:
|
||||||
|
try:
|
||||||
|
response = self.client.put_records(Records=events, StreamName=data_stream)
|
||||||
|
logger.info('Succesfully sended')
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning('Exception has occurred {}, retrying...'.format(e))
|
||||||
|
attempt += 1
|
||||||
|
time.sleep(attempt)
|
||||||
|
|
||||||
|
logger.error('Max attempt has been reached, rethrowing the last err')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_event(event):
|
||||||
|
"""
|
||||||
|
Events from CSV have no dtypes, lets convert it to some more real values (int / decimals etc)
|
||||||
|
:param event:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
msg_key = event["symbol"]
|
||||||
|
|
||||||
|
msg_formatted = {
|
||||||
|
"transaction_ts": int(event["transaction_ts"]),
|
||||||
|
"symbol": event["symbol"],
|
||||||
|
"price": float(event["price"]),
|
||||||
|
"amount": float(event["amount"]),
|
||||||
|
"dollar_amount": float(event["dollar_amount"]),
|
||||||
|
"type": event["type"],
|
||||||
|
"trans_id": int(event["trans_id"]),
|
||||||
|
}
|
||||||
|
|
||||||
|
return msg_formatted, msg_key
|
||||||
|
|
||||||
|
|
||||||
|
def produce_data(kinesis_data_stream, messages_per_sec, input_file, single_run):
|
||||||
|
"""
|
||||||
|
Main method for producing
|
||||||
|
:param kinesis_data_stream: param from cmdline name of KDS
|
||||||
|
:param messages_per_sec: param from cmdline max speed per sec 1/mps
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
kp = KinesisProducer(speed_per_sec=messages_per_sec)
|
||||||
|
|
||||||
|
with open(input_file) as csv_file:
|
||||||
|
|
||||||
|
events = []
|
||||||
|
|
||||||
|
reader = csv.DictReader(csv_file, delimiter=',')
|
||||||
|
all_rows = list(reader)
|
||||||
|
|
||||||
|
current_time = int(all_rows[0]["transaction_ts"])
|
||||||
|
|
||||||
|
replay_cnt = 1
|
||||||
|
events_cnt = 0
|
||||||
|
|
||||||
|
while True:
|
||||||
|
logger.info("start replaying for the {} time".format(replay_cnt))
|
||||||
|
for row in all_rows:
|
||||||
|
|
||||||
|
new_event_time = int(row["transaction_ts"])
|
||||||
|
time_delta = new_event_time - current_time
|
||||||
|
current_time = new_event_time
|
||||||
|
|
||||||
|
if time_delta > 0 and messages_per_sec > 0:
|
||||||
|
time.sleep(time_delta / messages_per_sec)
|
||||||
|
|
||||||
|
event, key = prepare_event(row)
|
||||||
|
# event_prepared = str(event)
|
||||||
|
data_for_event = {'Data':str(event), 'PartitionKey':key}
|
||||||
|
events.append(data_for_event)
|
||||||
|
events_cnt+=1
|
||||||
|
print(events_cnt)
|
||||||
|
if events_cnt == 10:
|
||||||
|
kp.produce(events, kinesis_data_stream)
|
||||||
|
events = []
|
||||||
|
events_cnt=0
|
||||||
|
time.sleep(1)
|
||||||
|
if single_run:
|
||||||
|
break
|
||||||
|
replay_cnt += 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
logger.info('Starting Simple Kinesis Producer (replaying stock data)')
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('-k', '--kinesis_ds', dest='kinesis_ds', required=True)
|
||||||
|
parser.add_argument('-i', '--input_file', dest='input_file', required=False)
|
||||||
|
parser.add_argument('-s', '--messages_per_sec', dest='mps', type=int, default=-1, required=False)
|
||||||
|
parser.add_argument('-r', '--single-run', dest='singel_run', action='store_true', required=False, default=False)
|
||||||
|
|
||||||
|
args, unknown = parser.parse_known_args()
|
||||||
|
config = configparser.ConfigParser()
|
||||||
|
|
||||||
|
kinesis_data_stream = args.kinesis_ds
|
||||||
|
messages_per_sec = int(args.mps)
|
||||||
|
|
||||||
|
single_run = args.singel_run if hasattr(args, 'singel_run') else False
|
||||||
|
|
||||||
|
if args.input_file:
|
||||||
|
input_file = args.input_file
|
||||||
|
else:
|
||||||
|
main_path = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
input_file = os.path.join(main_path, DEFAULT_DATA_FILE)
|
||||||
|
|
||||||
|
produce_data(kinesis_data_stream, messages_per_sec, input_file, single_run)
|
||||||
@ -1,9 +1,327 @@
|
|||||||
{
|
{
|
||||||
"version": 4,
|
"version": 4,
|
||||||
"terraform_version": "1.8.1",
|
"terraform_version": "1.8.1",
|
||||||
"serial": 25,
|
"serial": 43,
|
||||||
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
|
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
|
||||||
"outputs": {},
|
"outputs": {},
|
||||||
"resources": [],
|
"resources": [
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_glue_catalog_database",
|
||||||
|
"name": "datalake_db_raw_zone",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680",
|
||||||
|
"catalog_id": "534534002841",
|
||||||
|
"create_table_default_permission": [
|
||||||
|
{
|
||||||
|
"permissions": [
|
||||||
|
"ALL"
|
||||||
|
],
|
||||||
|
"principal": [
|
||||||
|
{
|
||||||
|
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "",
|
||||||
|
"federated_database": [],
|
||||||
|
"id": "534534002841:datalake_raw_534534002841_ab_1201680",
|
||||||
|
"location_uri": "",
|
||||||
|
"name": "datalake_raw_534534002841_ab_1201680",
|
||||||
|
"parameters": null,
|
||||||
|
"tags": null,
|
||||||
|
"tags_all": {},
|
||||||
|
"target_database": []
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "bnVsbA=="
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_kinesis_firehose_delivery_stream",
|
||||||
|
"name": "stock_delivery_stream",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
|
||||||
|
"destination": "extended_s3",
|
||||||
|
"destination_id": "destinationId-000000000001",
|
||||||
|
"elasticsearch_configuration": [],
|
||||||
|
"extended_s3_configuration": [
|
||||||
|
{
|
||||||
|
"bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
|
||||||
|
"buffering_interval": 60,
|
||||||
|
"buffering_size": 1,
|
||||||
|
"cloudwatch_logging_options": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"log_group_name": "",
|
||||||
|
"log_stream_name": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"compression_format": "UNCOMPRESSED",
|
||||||
|
"custom_time_zone": "UTC",
|
||||||
|
"data_format_conversion_configuration": [],
|
||||||
|
"dynamic_partitioning_configuration": [],
|
||||||
|
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||||
|
"file_extension": "",
|
||||||
|
"kms_key_arn": "",
|
||||||
|
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||||
|
"processing_configuration": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"processors": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"role_arn": "arn:aws:iam::534534002841:role/LabRole",
|
||||||
|
"s3_backup_configuration": [],
|
||||||
|
"s3_backup_mode": "Disabled"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"http_endpoint_configuration": [],
|
||||||
|
"id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
|
||||||
|
"kinesis_source_configuration": [
|
||||||
|
{
|
||||||
|
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
|
||||||
|
"role_arn": "arn:aws:iam::534534002841:role/LabRole"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"msk_source_configuration": [],
|
||||||
|
"name": "firehose-534534002841-ab-1201680",
|
||||||
|
"opensearch_configuration": [],
|
||||||
|
"opensearchserverless_configuration": [],
|
||||||
|
"redshift_configuration": [],
|
||||||
|
"server_side_encryption": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"key_arn": "",
|
||||||
|
"key_type": "AWS_OWNED_CMK"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"snowflake_configuration": [],
|
||||||
|
"splunk_configuration": [],
|
||||||
|
"tags": null,
|
||||||
|
"tags_all": {},
|
||||||
|
"timeouts": null,
|
||||||
|
"version_id": "1"
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
|
||||||
|
"dependencies": [
|
||||||
|
"aws_kinesis_stream.cryptostock_stream",
|
||||||
|
"aws_s3_bucket.raw_bucket"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_kinesis_stream",
|
||||||
|
"name": "cryptostock_stream",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 1,
|
||||||
|
"attributes": {
|
||||||
|
"arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
|
||||||
|
"encryption_type": "NONE",
|
||||||
|
"enforce_consumer_deletion": true,
|
||||||
|
"id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
|
||||||
|
"kms_key_id": "",
|
||||||
|
"name": "cryptostock-534534002841-ab-1201680",
|
||||||
|
"retention_period": 24,
|
||||||
|
"shard_count": 1,
|
||||||
|
"shard_level_metrics": [
|
||||||
|
"IncomingBytes",
|
||||||
|
"IncomingRecords",
|
||||||
|
"OutgoingBytes",
|
||||||
|
"OutgoingRecords"
|
||||||
|
],
|
||||||
|
"stream_mode_details": [
|
||||||
|
{
|
||||||
|
"stream_mode": "PROVISIONED"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Owner": "Aron Boguszewski",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Owner": "Aron Boguszewski",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"timeouts": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_s3_bucket",
|
||||||
|
"name": "processed_bucket",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"acceleration_status": "",
|
||||||
|
"acl": null,
|
||||||
|
"arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680",
|
||||||
|
"bucket": "datalake-processed-534534002841-ab-1201680",
|
||||||
|
"bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com",
|
||||||
|
"bucket_prefix": "",
|
||||||
|
"bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
|
||||||
|
"cors_rule": [],
|
||||||
|
"force_destroy": true,
|
||||||
|
"grant": [
|
||||||
|
{
|
||||||
|
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
|
||||||
|
"permissions": [
|
||||||
|
"FULL_CONTROL"
|
||||||
|
],
|
||||||
|
"type": "CanonicalUser",
|
||||||
|
"uri": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||||
|
"id": "datalake-processed-534534002841-ab-1201680",
|
||||||
|
"lifecycle_rule": [],
|
||||||
|
"logging": [],
|
||||||
|
"object_lock_configuration": [],
|
||||||
|
"object_lock_enabled": false,
|
||||||
|
"policy": "",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"replication_configuration": [],
|
||||||
|
"request_payer": "BucketOwner",
|
||||||
|
"server_side_encryption_configuration": [
|
||||||
|
{
|
||||||
|
"rule": [
|
||||||
|
{
|
||||||
|
"apply_server_side_encryption_by_default": [
|
||||||
|
{
|
||||||
|
"kms_master_key_id": "",
|
||||||
|
"sse_algorithm": "AES256"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"bucket_key_enabled": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"timeouts": null,
|
||||||
|
"versioning": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"mfa_delete": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"website": [],
|
||||||
|
"website_domain": null,
|
||||||
|
"website_endpoint": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"mode": "managed",
|
||||||
|
"type": "aws_s3_bucket",
|
||||||
|
"name": "raw_bucket",
|
||||||
|
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||||
|
"instances": [
|
||||||
|
{
|
||||||
|
"schema_version": 0,
|
||||||
|
"attributes": {
|
||||||
|
"acceleration_status": "",
|
||||||
|
"acl": null,
|
||||||
|
"arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
|
||||||
|
"bucket": "datalake-raw-534534002841-ab-1201680",
|
||||||
|
"bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com",
|
||||||
|
"bucket_prefix": "",
|
||||||
|
"bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
|
||||||
|
"cors_rule": [],
|
||||||
|
"force_destroy": true,
|
||||||
|
"grant": [
|
||||||
|
{
|
||||||
|
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
|
||||||
|
"permissions": [
|
||||||
|
"FULL_CONTROL"
|
||||||
|
],
|
||||||
|
"type": "CanonicalUser",
|
||||||
|
"uri": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||||
|
"id": "datalake-raw-534534002841-ab-1201680",
|
||||||
|
"lifecycle_rule": [],
|
||||||
|
"logging": [],
|
||||||
|
"object_lock_configuration": [],
|
||||||
|
"object_lock_enabled": false,
|
||||||
|
"policy": "",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"replication_configuration": [],
|
||||||
|
"request_payer": "BucketOwner",
|
||||||
|
"server_side_encryption_configuration": [
|
||||||
|
{
|
||||||
|
"rule": [
|
||||||
|
{
|
||||||
|
"apply_server_side_encryption_by_default": [
|
||||||
|
{
|
||||||
|
"kms_master_key_id": "",
|
||||||
|
"sse_algorithm": "AES256"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"bucket_key_enabled": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"tags_all": {
|
||||||
|
"Environment": "DEV",
|
||||||
|
"Purpose": "UAM Cloud Data Processing"
|
||||||
|
},
|
||||||
|
"timeouts": null,
|
||||||
|
"versioning": [
|
||||||
|
{
|
||||||
|
"enabled": false,
|
||||||
|
"mfa_delete": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"website": [],
|
||||||
|
"website_domain": null,
|
||||||
|
"website_endpoint": null
|
||||||
|
},
|
||||||
|
"sensitive_attributes": [],
|
||||||
|
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
"check_results": null
|
"check_results": null
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,327 +1,9 @@
|
|||||||
{
|
{
|
||||||
"version": 4,
|
"version": 4,
|
||||||
"terraform_version": "1.8.1",
|
"terraform_version": "1.8.1",
|
||||||
"serial": 19,
|
"serial": 37,
|
||||||
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
|
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
|
||||||
"outputs": {},
|
"outputs": {},
|
||||||
"resources": [
|
"resources": [],
|
||||||
{
|
|
||||||
"mode": "managed",
|
|
||||||
"type": "aws_glue_catalog_database",
|
|
||||||
"name": "datalake_db_raw_zone",
|
|
||||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
|
||||||
"instances": [
|
|
||||||
{
|
|
||||||
"schema_version": 0,
|
|
||||||
"attributes": {
|
|
||||||
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680",
|
|
||||||
"catalog_id": "534534002841",
|
|
||||||
"create_table_default_permission": [
|
|
||||||
{
|
|
||||||
"permissions": [
|
|
||||||
"ALL"
|
|
||||||
],
|
|
||||||
"principal": [
|
|
||||||
{
|
|
||||||
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"description": "",
|
|
||||||
"federated_database": [],
|
|
||||||
"id": "534534002841:datalake_raw_534534002841_ab_1201680",
|
|
||||||
"location_uri": "",
|
|
||||||
"name": "datalake_raw_534534002841_ab_1201680",
|
|
||||||
"parameters": null,
|
|
||||||
"tags": null,
|
|
||||||
"tags_all": {},
|
|
||||||
"target_database": []
|
|
||||||
},
|
|
||||||
"sensitive_attributes": [],
|
|
||||||
"private": "bnVsbA=="
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"mode": "managed",
|
|
||||||
"type": "aws_kinesis_firehose_delivery_stream",
|
|
||||||
"name": "stock_delivery_stream",
|
|
||||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
|
||||||
"instances": [
|
|
||||||
{
|
|
||||||
"schema_version": 1,
|
|
||||||
"attributes": {
|
|
||||||
"arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
|
|
||||||
"destination": "extended_s3",
|
|
||||||
"destination_id": "destinationId-000000000001",
|
|
||||||
"elasticsearch_configuration": [],
|
|
||||||
"extended_s3_configuration": [
|
|
||||||
{
|
|
||||||
"bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
|
|
||||||
"buffering_interval": 60,
|
|
||||||
"buffering_size": 1,
|
|
||||||
"cloudwatch_logging_options": [
|
|
||||||
{
|
|
||||||
"enabled": false,
|
|
||||||
"log_group_name": "",
|
|
||||||
"log_stream_name": ""
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"compression_format": "UNCOMPRESSED",
|
|
||||||
"custom_time_zone": "UTC",
|
|
||||||
"data_format_conversion_configuration": [],
|
|
||||||
"dynamic_partitioning_configuration": [],
|
|
||||||
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
|
||||||
"file_extension": "",
|
|
||||||
"kms_key_arn": "",
|
|
||||||
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
|
||||||
"processing_configuration": [
|
|
||||||
{
|
|
||||||
"enabled": false,
|
|
||||||
"processors": []
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"role_arn": "arn:aws:iam::534534002841:role/LabRole",
|
|
||||||
"s3_backup_configuration": [],
|
|
||||||
"s3_backup_mode": "Disabled"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"http_endpoint_configuration": [],
|
|
||||||
"id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
|
|
||||||
"kinesis_source_configuration": [
|
|
||||||
{
|
|
||||||
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
|
|
||||||
"role_arn": "arn:aws:iam::534534002841:role/LabRole"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"msk_source_configuration": [],
|
|
||||||
"name": "firehose-534534002841-ab-1201680",
|
|
||||||
"opensearch_configuration": [],
|
|
||||||
"opensearchserverless_configuration": [],
|
|
||||||
"redshift_configuration": [],
|
|
||||||
"server_side_encryption": [
|
|
||||||
{
|
|
||||||
"enabled": false,
|
|
||||||
"key_arn": "",
|
|
||||||
"key_type": "AWS_OWNED_CMK"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"snowflake_configuration": [],
|
|
||||||
"splunk_configuration": [],
|
|
||||||
"tags": {},
|
|
||||||
"tags_all": {},
|
|
||||||
"timeouts": null,
|
|
||||||
"version_id": "1"
|
|
||||||
},
|
|
||||||
"sensitive_attributes": [],
|
|
||||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
|
|
||||||
"dependencies": [
|
|
||||||
"aws_kinesis_stream.cryptostock_stream",
|
|
||||||
"aws_s3_bucket.raw_bucket"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"mode": "managed",
|
|
||||||
"type": "aws_kinesis_stream",
|
|
||||||
"name": "cryptostock_stream",
|
|
||||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
|
||||||
"instances": [
|
|
||||||
{
|
|
||||||
"schema_version": 1,
|
|
||||||
"attributes": {
|
|
||||||
"arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
|
|
||||||
"encryption_type": "NONE",
|
|
||||||
"enforce_consumer_deletion": true,
|
|
||||||
"id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
|
|
||||||
"kms_key_id": "",
|
|
||||||
"name": "cryptostock-534534002841-ab-1201680",
|
|
||||||
"retention_period": 24,
|
|
||||||
"shard_count": 1,
|
|
||||||
"shard_level_metrics": [
|
|
||||||
"IncomingBytes",
|
|
||||||
"IncomingRecords",
|
|
||||||
"OutgoingBytes",
|
|
||||||
"OutgoingRecords"
|
|
||||||
],
|
|
||||||
"stream_mode_details": [
|
|
||||||
{
|
|
||||||
"stream_mode": "PROVISIONED"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"tags": {
|
|
||||||
"Environment": "DEV",
|
|
||||||
"Owner": "Aron Boguszewski",
|
|
||||||
"Purpose": "UAM Cloud Data Processing"
|
|
||||||
},
|
|
||||||
"tags_all": {
|
|
||||||
"Environment": "DEV",
|
|
||||||
"Owner": "Aron Boguszewski",
|
|
||||||
"Purpose": "UAM Cloud Data Processing"
|
|
||||||
},
|
|
||||||
"timeouts": null
|
|
||||||
},
|
|
||||||
"sensitive_attributes": [],
|
|
||||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"mode": "managed",
|
|
||||||
"type": "aws_s3_bucket",
|
|
||||||
"name": "processed_bucket",
|
|
||||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
|
||||||
"instances": [
|
|
||||||
{
|
|
||||||
"schema_version": 0,
|
|
||||||
"attributes": {
|
|
||||||
"acceleration_status": "",
|
|
||||||
"acl": null,
|
|
||||||
"arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680",
|
|
||||||
"bucket": "datalake-processed-534534002841-ab-1201680",
|
|
||||||
"bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com",
|
|
||||||
"bucket_prefix": "",
|
|
||||||
"bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
|
|
||||||
"cors_rule": [],
|
|
||||||
"force_destroy": true,
|
|
||||||
"grant": [
|
|
||||||
{
|
|
||||||
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
|
|
||||||
"permissions": [
|
|
||||||
"FULL_CONTROL"
|
|
||||||
],
|
|
||||||
"type": "CanonicalUser",
|
|
||||||
"uri": ""
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
|
||||||
"id": "datalake-processed-534534002841-ab-1201680",
|
|
||||||
"lifecycle_rule": [],
|
|
||||||
"logging": [],
|
|
||||||
"object_lock_configuration": [],
|
|
||||||
"object_lock_enabled": false,
|
|
||||||
"policy": "",
|
|
||||||
"region": "us-east-1",
|
|
||||||
"replication_configuration": [],
|
|
||||||
"request_payer": "BucketOwner",
|
|
||||||
"server_side_encryption_configuration": [
|
|
||||||
{
|
|
||||||
"rule": [
|
|
||||||
{
|
|
||||||
"apply_server_side_encryption_by_default": [
|
|
||||||
{
|
|
||||||
"kms_master_key_id": "",
|
|
||||||
"sse_algorithm": "AES256"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"bucket_key_enabled": false
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"tags": {
|
|
||||||
"Environment": "DEV",
|
|
||||||
"Purpose": "UAM Cloud Data Processing"
|
|
||||||
},
|
|
||||||
"tags_all": {
|
|
||||||
"Environment": "DEV",
|
|
||||||
"Purpose": "UAM Cloud Data Processing"
|
|
||||||
},
|
|
||||||
"timeouts": null,
|
|
||||||
"versioning": [
|
|
||||||
{
|
|
||||||
"enabled": false,
|
|
||||||
"mfa_delete": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"website": [],
|
|
||||||
"website_domain": null,
|
|
||||||
"website_endpoint": null
|
|
||||||
},
|
|
||||||
"sensitive_attributes": [],
|
|
||||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"mode": "managed",
|
|
||||||
"type": "aws_s3_bucket",
|
|
||||||
"name": "raw_bucket",
|
|
||||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
|
||||||
"instances": [
|
|
||||||
{
|
|
||||||
"schema_version": 0,
|
|
||||||
"attributes": {
|
|
||||||
"acceleration_status": "",
|
|
||||||
"acl": null,
|
|
||||||
"arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
|
|
||||||
"bucket": "datalake-raw-534534002841-ab-1201680",
|
|
||||||
"bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com",
|
|
||||||
"bucket_prefix": "",
|
|
||||||
"bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
|
|
||||||
"cors_rule": [],
|
|
||||||
"force_destroy": true,
|
|
||||||
"grant": [
|
|
||||||
{
|
|
||||||
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
|
|
||||||
"permissions": [
|
|
||||||
"FULL_CONTROL"
|
|
||||||
],
|
|
||||||
"type": "CanonicalUser",
|
|
||||||
"uri": ""
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
|
||||||
"id": "datalake-raw-534534002841-ab-1201680",
|
|
||||||
"lifecycle_rule": [],
|
|
||||||
"logging": [],
|
|
||||||
"object_lock_configuration": [],
|
|
||||||
"object_lock_enabled": false,
|
|
||||||
"policy": "",
|
|
||||||
"region": "us-east-1",
|
|
||||||
"replication_configuration": [],
|
|
||||||
"request_payer": "BucketOwner",
|
|
||||||
"server_side_encryption_configuration": [
|
|
||||||
{
|
|
||||||
"rule": [
|
|
||||||
{
|
|
||||||
"apply_server_side_encryption_by_default": [
|
|
||||||
{
|
|
||||||
"kms_master_key_id": "",
|
|
||||||
"sse_algorithm": "AES256"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"bucket_key_enabled": false
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"tags": {
|
|
||||||
"Environment": "DEV",
|
|
||||||
"Purpose": "UAM Cloud Data Processing"
|
|
||||||
},
|
|
||||||
"tags_all": {
|
|
||||||
"Environment": "DEV",
|
|
||||||
"Purpose": "UAM Cloud Data Processing"
|
|
||||||
},
|
|
||||||
"timeouts": null,
|
|
||||||
"versioning": [
|
|
||||||
{
|
|
||||||
"enabled": false,
|
|
||||||
"mfa_delete": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"website": [],
|
|
||||||
"website_domain": null,
|
|
||||||
"website_endpoint": null
|
|
||||||
},
|
|
||||||
"sensitive_attributes": [],
|
|
||||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"check_results": null
|
"check_results": null
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user