commit e1cd70aab3460c342d219816d4d5cf3aae7b305e Author: Marcin Czerniak Date: Thu Feb 15 15:13:18 2024 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c569c36 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +gitea-key.pem \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..3025f66 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# Gitea uruchamiana automatycznie na środowisku AWS + +Zadania numer `3.1` i `3.2` z przedmiotu *Chmury obliczeniowe* zrealizowane na środowisku AWS. + +## Wymagania wstępne +- Konto na platformie AWS +- skonfigurowany plik `.aws/credentials` z danymi dostępowymi do AWS + +## Uruchomienie +1. Sklonuj repozytorium +2. Uruchom skrypt `deploy.sh`: +```bash +./deploy.sh +``` + +Opcjonalnie można uruchomić skrypt `deploy.py`: +```bash +python3 deploy.py +``` + +## Zasada działania +Skrypt `deploy.py` tworzy sieć VPC, subnety, routing table, security group, 2 instancje EC2 i dodatkowy dysk twardy. Następnie instaluje na jednej z instancji serwer Gitea, a na drugiej bazę danych PostgreSQL (na obu instancjach aplikację działają na dockerze i komunikują się między sobą). + +Skrypt `deploy.sh` uruchamia skrypt `deploy.py`. \ No newline at end of file diff --git a/deploy.py b/deploy.py new file mode 100644 index 0000000..2d07aa3 --- /dev/null +++ b/deploy.py @@ -0,0 +1,370 @@ +import boto3 +import time +import subprocess +import platform +import paramiko +import string +import random + + +REGION = 'us-east-1' + +# -------------------------------------------------- +# ----------------- AWS Connection ----------------- +# -------------------------------------------------- + +def check_aws_connection(): + try: + s3 = boto3.client('s3') + s3.list_buckets() + print('AWS connection success.') + + except Exception as e: + print(f'Error: {e}') + print('AWS connection failed.') + +check_aws_connection() + +# -------------------------------------------------- +# --------------- Utility functions ---------------- +# -------------------------------------------------- + +def get_public_dns_name(inst): + client = boto3.client('ec2', region_name=REGION) + response = client.describe_instances(InstanceIds = [inst.id]) + return response['Reservations'][0]['Instances'][0]['NetworkInterfaces'][0]['Association']['PublicDnsName'] + +def get_private_dns_name(inst): + client = boto3.client('ec2', region_name=REGION) + response = client.describe_instances(InstanceIds = [inst.id]) + return response['Reservations'][0]['Instances'][0]['NetworkInterfaces'][0]['PrivateDnsName'] + +def random_string(length): + letters = string.ascii_lowercase + return ''.join(random.choice(letters) for i in range(length)) + +# -------------------------------------------------- +# ----------------- Create key pair ---------------- +# -------------------------------------------------- + +def create_key_pair(key_name): + ec2 = boto3.client('ec2', region_name=REGION) + key_pair = ec2.create_key_pair(KeyName=key_name) + + with open(f'{key_name}.pem', 'w') as file: + file.write(key_pair['KeyMaterial']) + + return key_pair + +# -------------------------------------------------- +# ------------------ Networking -------------------- +# -------------------------------------------------- + +def get_default_vpc(): + ec2 = boto3.client('ec2', region_name=REGION) + response = ec2.describe_vpcs() + vpc_id = response['Vpcs'][0]['VpcId'] + return vpc_id + +def create_vpc(): + ec2 = boto3.resource('ec2', region_name=REGION) + vpc = ec2.create_vpc(CidrBlock='172.31.0.0/16') + + vpc.modify_attribute(EnableDnsSupport={'Value': True}) + vpc.modify_attribute(EnableDnsHostnames={'Value': True}) + + igw = ec2.create_internet_gateway() + vpc.attach_internet_gateway(InternetGatewayId=igw.id) + + return vpc, igw + +def create_subnet(vpc, cidr_block, availability_zone): + subnet = vpc.create_subnet( + CidrBlock=cidr_block, + AvailabilityZone=availability_zone + ) + + route_table = vpc.create_route_table() + route_table.associate_with_subnet(SubnetId=subnet.id) + + route_table.create_route( + DestinationCidrBlock='0.0.0.0/0', + GatewayId=igw.id + ) + + return subnet, route_table + +def create_security_group(vpc, group_name, description): + ec2 = boto3.resource('ec2', region_name=REGION) + security_group = ec2.create_security_group( + GroupName=group_name, + Description=description, + VpcId=vpc.id + ) + + security_group.authorize_ingress( + CidrIp='0.0.0.0/0', + IpProtocol='tcp', + FromPort=22, + ToPort=22 + ) + + security_group.authorize_ingress( + CidrIp='0.0.0.0/0', + IpProtocol='tcp', + FromPort=80, + ToPort=80 + ) + + security_group.authorize_ingress( + CidrIp='0.0.0.0/0', + IpProtocol='tcp', + FromPort=443, + ToPort=443 + ) + + security_group.authorize_ingress( + CidrIp='172.31.0.0/16', + IpProtocol='tcp', + FromPort=5432, + ToPort=5432 + ) + + return security_group + +# -------------------------------------------------- +# ------------ AWS Machine reservation ------------- +# -------------------------------------------------- + +def create_ec2_instance(ami_id, instance_name, security_group_id, subnet_id, key_pair, with_volume=False): + ec2 = boto3.resource('ec2', region_name=REGION) + + instance = ec2.create_instances( + ImageId=ami_id, + InstanceType='t2.micro', + MinCount=1, + MaxCount=1, + NetworkInterfaces=[{ + 'SubnetId': subnet_id, + 'DeviceIndex': 0, + 'AssociatePublicIpAddress': True, + 'DeleteOnTermination': True, + 'Groups': [security_group_id] + }], + TagSpecifications=[{'ResourceType': 'instance', 'Tags': [{'Key': 'Name', 'Value': instance_name}]}], + KeyName=key_pair['KeyName'] + )[0] + + instance.wait_until_running() + + if with_volume: + volume = ec2.create_volume( + AvailabilityZone=instance.placement['AvailabilityZone'], + Size=8, + VolumeType='gp2', + TagSpecifications=[{'ResourceType': 'volume', 'Tags': [{'Key': 'Name', 'Value': f'{instance_name}-volume'}]}] + ) + + counter = 10 + while counter > 0: + time.sleep(5) + try: + instance.attach_volume( + Device='/dev/sdf', + VolumeId=volume.id + ) + break + except Exception as e: + if (counter == 1): + print(f"Error: {e}") + counter -= 1 + + return instance + + +if __name__ == "__main__": + try: + ami_id = 'ami-0ee3dd41c47751fe6' + + # Create a VPC + print("[Stage 1/8] Creating VPC...") + vpc, igw = create_vpc() + + # Create security group for the instances + print("[Stage 2/8] Creating security group...") + security_group = create_security_group(vpc, 'GiteaSecurityGroup', 'Gitea Security Group') + + # Create subnets within the VPC for each instance + print("[Stage 3/8] Creating subnets...") + subnet, route_table = create_subnet(vpc, '172.31.2.0/16', REGION + 'a') + + # Create a key pair for SSH access + print("[Stage 4/8] Creating key pair...") + key_pair = create_key_pair('gitea-key') + + # Create two EC2 instances within their respective subnets + print("[Stage 5/8] Creating EC2 instances (POSTGRES)...") + postgres_instance = create_ec2_instance(ami_id, 'PostgresInstance', security_group.group_id, subnet.id, key_pair) + print("[Stage 6/8] Creating EC2 instances (GITEA)...") + gitea_instance = create_ec2_instance(ami_id, 'GiteaInstance', security_group.group_id, subnet.id, key_pair, True) + + # Get public DNS of the instances + gitea_public_dns = get_public_dns_name(gitea_instance) + postgres_public_dns = get_public_dns_name(postgres_instance) + postgres_private_dns = postgres_instance.private_dns_name + + # Print public DNS of the instances + print(f"Postgres instance public DNS: {postgres_public_dns}") + print(f"Gitea instance public DNS: {gitea_public_dns}") + + db_password = random_string(16) + + # ssh into the instance and install PostgreSQL + print("[Stage 7/8] Configuring Postgres instance...") + counter = 5 + + while counter > 0: + try: + print(f"Trying to connect to {postgres_public_dns}...") + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(postgres_public_dns, username='ec2-user', key_filename=f'{key_pair["KeyName"]}.pem') + break + except Exception as e: + print(f"Error: {e}") + print("Retrying...") + time.sleep(5) + counter -= 1 + + print("Connected to Postgres instance.") + stdin, stdout, stderr = ssh_client.exec_command('sudo yum update -y') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo yum install -y docker') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo service docker start') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo mkdir postgres-dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo touch postgres-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo chmod 777 postgres-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "FROM postgres:latest" >> postgres-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "ENV POSTGRES_USER gitea" >> postgres-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command(f'sudo echo "ENV POSTGRES_PASSWORD {db_password}" >> postgres-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "ENV POSTGRES_DB gitea" >> postgres-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo docker build -t postgres postgres-dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command(f'sudo docker run -d -p 5432:5432 postgres') + + + # Configure gitea instance + print("[Stage 8/8] Configuring Gitea instance...") + counter = 5 + + while counter > 0: + try: + print(f"Trying to connect to {gitea_public_dns}...") + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(gitea_public_dns, username='ec2-user', key_filename=f'{key_pair["KeyName"]}.pem') + break + except Exception as e: + print(f"Error: {e}") + print("Retrying...") + time.sleep(5) + counter -= 1 + + print("Connected to Gitea instance.") + stdin, stdout, stderr = ssh_client.exec_command('sudo yum update -y') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo mkfs.ext4 /dev/xvdf') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo mkdir /data') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo mount /dev/xvdf /data') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo mkdir /data/gitea') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo yum install -y docker') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo service docker start') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo mkdir gitea-dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo touch gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo chmod 777 gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "FROM gitea/gitea:latest" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "ENV GITEA__database__DB_TYPE postgres" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command(f'sudo echo "ENV GITEA__database__HOST {postgres_private_dns}:5432" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "ENV GITEA__database__NAME gitea" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "ENV GITEA__database__USER gitea" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command(f'sudo echo "ENV GITEA__database__PASSWD {db_password}" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "ENV GITEA__server__APP_DATA_PATH /data/gitea" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "RUN mkdir /data/gitea" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo echo "EXPOSE 3000" >> gitea-dockerfile/Dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo docker build -t gitea gitea-dockerfile') + print(stdout.read().decode()) + stdin, stdout, stderr = ssh_client.exec_command('sudo docker run -d -p 80:3000 -v /data/gitea:/data/gitea gitea') + print(stdout.read().decode()) + + print("Gitea and Postgres instances are ready to use.") + print(f"Website is available at: http://{gitea_public_dns}") + + finally: + # Wait for user input to destroy resources + input("Press Enter to destroy resources...") + + # Cleanup resources + if 'gitea_instance' in locals(): + print("Terminating Gitea instance...") + gitea_instance.terminate() + gitea_instance.wait_until_terminated() + + if 'postgres_instance' in locals(): + print("Terminating Postgres instance...") + postgres_instance.terminate() + postgres_instance.wait_until_terminated() + + if 'subnet' in locals(): + print("Deleting subnet...") + subnet.delete() + + if 'route_table' in locals(): + print("Deleting route table...") + route_table.delete() + + if 'security_group' in locals(): + print("Deleting security group...") + security_group.delete() + + if 'igw' in locals(): + print("Detaching and deleting Internet Gateway...") + igw.detach_from_vpc(VpcId=vpc.id) + igw.delete() + + if 'vpc' in locals(): + print("Deleting VPC...") + vpc.delete() + + if 'key_pair' in locals(): + print("Deleting key pair...") + client = boto3.client('ec2', region_name=REGION) + client.delete_key_pair(KeyName=key_pair['KeyName']) + + print("Resources destroyed.") \ No newline at end of file diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..f59bc42 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if ! command -v python3 &> /dev/null; then + if ! command -v python &> /dev/null; then + echo "Python3 is not installed" + else + python deploy.py + fi +else + python3 deploy.py +fi \ No newline at end of file