From 23f48f85cd8447596cbb604d4ba9dbf60cd632c2 Mon Sep 17 00:00:00 2001 From: Sebastian Rieger Date: Tue, 26 Apr 2022 23:39:38 +0200 Subject: [PATCH] =?UTF-8?q?added=20image=20search,=20expand=20user=20home?= =?UTF-8?q?=20dir=20based=20on=20platform,=20removed=20elastic=20ips=20to?= =?UTF-8?q?=20save=20budget,=20load=20AWS=20credentials=20file,=20fixed=20?= =?UTF-8?q?=C3=93OM=20in=20AWS=20m1.tiny=20when=20installing=20faafo=20req?= =?UTF-8?q?s=20using=20pip,=20increased=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../demo3-microservice-in-aws-destroy.py | 49 ++++-- example-projects/demo3-microservice-in-aws.py | 148 ++++++++++-------- 2 files changed, 119 insertions(+), 78 deletions(-) diff --git a/example-projects/demo3-microservice-in-aws-destroy.py b/example-projects/demo3-microservice-in-aws-destroy.py index c8c9339..b37f498 100644 --- a/example-projects/demo3-microservice-in-aws-destroy.py +++ b/example-projects/demo3-microservice-in-aws-destroy.py @@ -1,4 +1,6 @@ -import getpass +import configparser +from os.path import expanduser + # import os # import libcloud.security @@ -6,6 +8,8 @@ import time from libcloud.compute.providers import get_driver from libcloud.compute.types import Provider, NodeState +home = expanduser("~") + # reqs: # services: EC2 (nova, glance, neutron) # resources: 2 instances (m1.small), 2 elastic ips (1 keypair, 2 security groups) @@ -28,14 +32,18 @@ def main(): # ########################################################################### - # see AWS Educate classroom, Account Details + # see AWS Academy Lab for Account Details + # read credentials from file + config = configparser.ConfigParser() + config.read_file(open(home + '/.aws/credentials')) + aws_access_key_id = config['default']['aws_access_key_id'] + aws_secret_access_key = config['default']['aws_secret_access_key'] + aws_session_token = config['default']['aws_session_token'] + + # aws_access_key_id = "ASIAX..." + # aws_secret_access_key = "eGwE12j..." + # aws_session_token = "FwoGZXIvYXdzEK///////////wEaDE..." - # access_id = getpass.win_getpass("Enter your access_id:") - # secret_key = getpass.win_getpass("Enter your secret_key:") - # session_token = getpass.win_getpass("Enter your session_token:") - access_id = "ASIAX..." - secret_key = "eGwE12j..." - session_token = "FwoGZXIvYXdzEK///////////wEaDE..." ########################################################################### # @@ -44,9 +52,9 @@ def main(): ########################################################################### provider = get_driver(Provider.EC2) - conn = provider(access_id, - secret_key, - token=session_token, + conn = provider(aws_access_key_id, + aws_secret_access_key, + token=aws_session_token, region=region_name) ########################################################################### @@ -59,8 +67,9 @@ def main(): for instance in conn.list_nodes(): if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller', 'app-services', 'app-api-1', 'app-api-2']: - print('Destroying Instance: %s' % instance.name) - conn.destroy_node(instance) + if instance.state.value != 'terminated': + print('Destroying Instance: %s' % instance.name) + conn.destroy_node(instance) # wait until all nodes are destroyed to be able to remove depended security groups nodes_still_running = True @@ -78,7 +87,19 @@ def main(): # delete security groups for group in conn.ex_list_security_groups(): - if group in ['control', 'worker', 'api', 'services']: + # services depends on worker and api, so delete services first... + if group in ['services']: + print('Deleting security group: %s' % group) + conn.ex_delete_security_group(group) + + for group in conn.ex_list_security_groups(): + # control depends on worker, so delete control before worker... + if group in ['control']: + print('Deleting security group: %s' % group) + conn.ex_delete_security_group(group) + + for group in conn.ex_list_security_groups(): + if group in ['worker', 'api']: print('Deleting security group: %s' % group) conn.ex_delete_security_group(group) diff --git a/example-projects/demo3-microservice-in-aws.py b/example-projects/demo3-microservice-in-aws.py index 45901c3..1e5eb8e 100644 --- a/example-projects/demo3-microservice-in-aws.py +++ b/example-projects/demo3-microservice-in-aws.py @@ -1,16 +1,19 @@ -import getpass -# import os +import configparser +from os.path import expanduser + from libcloud.compute.base import NodeImage from libcloud.compute.providers import get_driver from libcloud.compute.types import Provider +home = expanduser("~") + # reqs: # services: EC2 (nova, glance, neutron) # resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups) # The image to look for and use for the started instance -ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408' -# ubuntu_image_id = "ami-0e342d72b12109f91" # local ami id for resent ubuntu 18.04 20200408 in eu-central-1 +ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128' +# TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3... # The public key to be used for SSH connection, please make sure, that you have the corresponding private key # @@ -18,7 +21,7 @@ ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-2020 # ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME keypair_name = 'srieger-pub' -pub_key_file = '~/.ssh/id_rsa.pub' +pub_key_file = home + '/.ssh/id_rsa.pub' flavor_name = 't2.nano' @@ -26,11 +29,12 @@ flavor_name = 't2.nano' # region_name = 'eu-central-1' # region_name = 'ap-south-1' -# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com -# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE - +# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/ region_name = 'us-east-1' +# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout +timeout = 300 + def main(): ########################################################################### @@ -39,14 +43,18 @@ def main(): # ########################################################################### - # see AWS Educate classroom, Account Details + # see AWS Academy Lab for Account Details + # read credentials from file + config = configparser.ConfigParser() + config.read_file(open(home + '/.aws/credentials')) + aws_access_key_id = config['default']['aws_access_key_id'] + aws_secret_access_key = config['default']['aws_secret_access_key'] + aws_session_token = config['default']['aws_session_token'] - # access_id = getpass.win_getpass("Enter your access_id:") - # secret_key = getpass.win_getpass("Enter your secret_key:") - # session_token = getpass.win_getpass("Enter your session_token:") - access_id = "ASIAX..." - secret_key = "WLxxXK+..." - session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F" + # hard coded AWS credentials using vars + # aws_access_key_id = "ASIAX..." + # aws_secret_access_key = "WLxxXK+..." + # aws_session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F" ########################################################################### # @@ -55,9 +63,9 @@ def main(): ########################################################################### provider = get_driver(Provider.EC2) - conn = provider(access_id, - secret_key, - token=session_token, + conn = provider(aws_access_key_id, + aws_secret_access_key, + token=aws_session_token, region=region_name) ########################################################################### @@ -66,10 +74,12 @@ def main(): # ########################################################################### + print("Search for AMI...") + image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0] + print("Using image: %s" % image) + # print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of " # "supported operating systems and versions)... please be patient...") - # images = conn.list_images() - # image = '' # for img in images: # # if img.name == ubuntu_image_name: @@ -79,7 +89,6 @@ def main(): # image = img # fetch/select the image referenced with ubuntu_image_name above - # image = [i for i in images if i.name == ubuntu_image_name][0] # print(image) @@ -88,9 +97,9 @@ def main(): # summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard, # to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the # desired region - image = NodeImage(id="ami-0e472ba40eb589f49", - name="ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20211129", - driver="hvm") + # image = NodeImage(id="ami-0e472ba40eb589f49", + # name=ubuntu_image_name, + # driver="hvm") flavors = conn.list_sizes() flavor = [s for s in flavors if s.id == flavor_name][0] @@ -183,14 +192,16 @@ def main(): # Thanks to Stefan Friedmann for finding this fix ;) userdata = '''#!/usr/bin/env bash - curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ + curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ -i messaging -i faafo -r api rabbitmqctl add_user faafo guest rabbitmqctl set_user_tags faafo administrator rabbitmqctl set_permissions -p / faafo ".*" ".*" ".*" ''' - print('Starting new app-controller instance and wait until it is running...') + print('Starting new app-controller instance and wait until it is running (can take several minutes in free AWS' + ' academy accounts compared to seconds when using a regular paid AWS account), timeout %i seconds...' + % timeout) instance_controller_1 = conn.create_node(name='app-controller', image=image, size=flavor, @@ -198,7 +209,8 @@ def main(): ex_userdata=userdata, ex_security_groups=[controller_security_group_name]) - conn.wait_until_running(nodes=[instance_controller_1], timeout=120, ssh_interface='public_ips') + wait_until_running_result = conn.wait_until_running(nodes=[instance_controller_1], timeout=timeout, ssh_interface='public_ips') + instance_controller_1, node_addresses = wait_until_running_result[0] ########################################################################### # @@ -206,20 +218,22 @@ def main(): # ########################################################################### - # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However elastic IPs cost money, - # and instances typically already have public IP in AWS, what a luxury ;) - print('Checking for unused Elastic IP...') - unused_elastic_ip = None - for elastic_ip in conn.ex_describe_all_addresses(): - if not elastic_ip.instance_id: - unused_elastic_ip = elastic_ip - break - - if not unused_elastic_ip: - print('Allocating new Elastic IP') - unused_elastic_ip = conn.ex_allocate_address() - conn.ex_associate_address_with_node(instance_controller_1, unused_elastic_ip) - print('Controller Application will be deployed to http://%s' % unused_elastic_ip.ip) + # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However, elastic IPs cost money, + # and instances typically already have public IP in AWS, what a luxury ;) so I commented out elastic IP creation to + # save your AWS academy budget + + # print('Checking for unused Elastic IP...') + # unused_elastic_ip = None + # for elastic_ip in conn.ex_describe_all_addresses(): + # if not elastic_ip.instance_id: + # unused_elastic_ip = elastic_ip + # break + # + # if not unused_elastic_ip: + # print('Allocating new Elastic IP') + # unused_elastic_ip = conn.ex_allocate_address() + # conn.ex_associate_address_with_node(instance_controller_1, unused_elastic_ip) + # print('Controller Application will be deployed to http://%s' % unused_elastic_ip.ip) ########################################################################### # @@ -227,12 +241,14 @@ def main(): # ########################################################################### - # instance should not have a public ip? floating ips are assigned later - # instance_controller_1 = conn.ex_get_node_details(instance_controller_1.id) - # if instance_controller_1.public_ips: - # ip_controller = instance_controller_1.public_ips[0] - # else: - ip_controller = instance_controller_1.private_ips[0] + # instance_controller_1 = conn.list_nodes(ex_node_ids=instance_controller_1.id) + public_ip_controller = instance_controller_1.public_ips[0] + print('Controller Application %s will be reachable after cloud-init has run at http://%s, you can also connect' + % (instance_controller_1.id, public_ip_controller) + + ' via ssh ubuntu@%s' % public_ip_controller) + + # get private IP of instance to use in worker/service instances to connect to the controller + private_ip_controller = instance_controller_1.private_ips[0] ########################################################################### # @@ -241,9 +257,9 @@ def main(): ########################################################################### userdata = '''#!/usr/bin/env bash - curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \ + curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \ -i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://faafo:guest@%(ip_controller)s:5672/' - ''' % {'ip_controller': ip_controller} + ''' % {'ip_controller': private_ip_controller} print('Starting new app-worker-1 instance and wait until it is running...') instance_worker_1 = conn.create_node(name='app-worker-1', @@ -253,7 +269,9 @@ def main(): ex_userdata=userdata, ex_security_groups=[worker_security_group_name]) - conn.wait_until_running(nodes=[instance_worker_1], timeout=120, ssh_interface='public_ips') + wait_until_running_result = conn.wait_until_running(nodes=[instance_worker_1], timeout=timeout, ssh_interface='public_ips') + instance_worker_1, node_addresses = wait_until_running_result[0] + print('Worker instance %s has private ip: %s ' % (instance_worker_1.id, instance_worker_1.private_ips[0])) ########################################################################### # @@ -261,20 +279,22 @@ def main(): # ########################################################################### - # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However elastic IPs cost money, - # and instances typically already have public IP in AWS, what a luxury ;) - print('Checking for unused Elastic IP...') - unused_elastic_ip = None - for elastic_ip in conn.ex_describe_all_addresses(): - if not elastic_ip.instance_id: - unused_elastic_ip = elastic_ip - break - - if not unused_elastic_ip: - print('Allocating new Elastic IP') - unused_elastic_ip = conn.ex_allocate_address() - conn.ex_associate_address_with_node(instance_worker_1, unused_elastic_ip) - print('The worker will be available for SSH at %s' % unused_elastic_ip.ip) + # AWS offers elastic ips, that have the same function as floating IPs in OpenStack. However, elastic IPs cost money, + # and instances typically already have public IP in AWS, what a luxury ;) so I commented out elastic IP creation to + # save your AWS academy budget + # print('Checking for unused Elastic IP...') + # unused_elastic_ip = None + # for elastic_ip in conn.ex_describe_all_addresses(): + # if not elastic_ip.instance_id: + # unused_elastic_ip = elastic_ip + # break + # + # if not unused_elastic_ip: + # print('Allocating new Elastic IP') + # unused_elastic_ip = conn.ex_allocate_address() + # conn.ex_associate_address_with_node(instance_worker_1, unused_elastic_ip) + # print('The worker will be available for SSH at %s' % unused_elastic_ip.ip) + # print('You can use ssh to login to the controller using your private key. After login, you can list available ' 'fractals using "faafo list". To request the generation of new fractals, you can use "faafo create". '