srieger
/
cloud-computing-msc-ai-examples
1
2
Fork 1
Code Issues Pull Requests 1 Releases Wiki Activity
Browse Source

updated demo4 to support AWS Academy lab setup

pull/3/head
Sebastian Rieger 3 years ago
parent
afd3bef5c2
commit
e18ab4dc70
4 changed files with 108 additions and 88 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      example-projects/demo3-microservice-in-aws-destroy.py
  2. 10
      example-projects/demo3-microservice-in-aws.py
  3. 83
      example-projects/demo4-scale-out-lb-in-aws-destroy.py
  4. 92
      example-projects/demo4-scale-out-lb-in-aws.py

11
example-projects/demo3-microservice-in-aws-destroy.py
View File

@ -10,20 +10,15 @@ from libcloud.compute.types import Provider, NodeState
home = expanduser("~")
# reqs:
# services: EC2 (nova, glance, neutron)
# resources: 2 instances (m1.small), 2 elastic ips (1 keypair, 2 security groups)
# default region
# region_name = 'eu-central-1'
# region_name = 'ap-south-1'
# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com
# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE
# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
region_name = 'us-east-1'
# keypairs are kept and not deleted, they do not cost anything anyway
def main():
###########################################################################

10
example-projects/demo3-microservice-in-aws.py
View File

@ -7,9 +7,11 @@ from libcloud.compute.types import Provider
home = expanduser("~")
# reqs:
# services: EC2 (nova, glance, neutron)
# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups)
# requirements:
# services: EC2
# resources: 2 instances (1 keypair, 2 security groups)
# optionally also elastic ip (comparable to floating ip) offering a persistent public
# IP, but elastic ips are expensive - make sure to delete them after you used them
# The image to look for and use for the started instance
ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
@ -33,7 +35,7 @@ flavor_name = 't2.nano'
region_name = 'us-east-1'
# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
timeout = 300
timeout = 600
def main():

83
example-projects/demo4-scale-out-lb-in-aws-destroy.py
View File

@ -1,3 +1,6 @@
import configparser
from os.path import expanduser
# import getpass
# import os
# import libcloud.security
@ -12,33 +15,17 @@ from libcloud.loadbalancer.base import Member, Algorithm
from libcloud.loadbalancer.types import Provider as loadbalancer_Provider
from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver
# reqs:
# services: EC2 (nova, glance, neutron)
# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups)
# The image to look for and use for the started instance
# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408'
ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 20200408 in us-west-1
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key
#
# id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub'
pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 't2.nano'
home = expanduser("~")
# default region
# region_name = 'eu-central-1'
# region_name = 'ap-south-1'
# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com
# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE
# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
region_name = 'us-east-1'
# keypairs are kept and not deleted, they do not cost anything anyway
def main():
###########################################################################
@ -47,17 +34,18 @@ def main():
#
###########################################################################
# see AWS Educate classroom, Account Details
# see AWS Academy Lab for Account Details
# read credentials from file
config = configparser.ConfigParser()
config.read_file(open(home + '/.aws/credentials'))
aws_access_key_id = config['default']['aws_access_key_id']
aws_secret_access_key = config['default']['aws_secret_access_key']
aws_session_token = config['default']['aws_session_token']
# aws_access_key_id = "ASIAX..."
# aws_secret_access_key = "eGwE12j..."
# aws_session_token = "FwoGZXIvYXdzEK///////////wEaDE..."
# access_id = getpass.win_getpass("Enter your access_id:")
# secret_key = getpass.win_getpass("Enter your secret_key:")
# session_token = getpass.win_getpass("Enter your session_token:")
# access_id = "ASIAU..."
# secret_key = "7lafW..."
# session_token = "IQoJb3JpZ...EMb//..."
access_id = "ASIA5ML7..."
secret_key = "76lAjn..."
session_token = "IQoJb3JpZ2luX2VjEBc..."
###########################################################################
#
@ -66,9 +54,9 @@ def main():
###########################################################################
elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB)
elb_conn = elb_provider(access_id,
secret_key,
token=session_token,
elb_conn = elb_provider(aws_access_key_id,
aws_secret_access_key,
token=aws_session_token,
region=region_name)
print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers()))
@ -83,9 +71,9 @@ def main():
###########################################################################
provider = compute_get_driver(compute_Provider.EC2)
conn = provider(key=access_id,
secret=secret_key,
token=session_token,
conn = provider(key=aws_access_key_id,
secret=aws_secret_access_key,
token=aws_session_token,
region=region_name)
###########################################################################
@ -102,7 +90,7 @@ def main():
print('Destroying Instance: %s' % instance.name)
conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups
# wait until all nodes are destroyed to be able to remove dependent security groups
nodes_still_running = True
while nodes_still_running:
nodes_still_running = False
@ -115,19 +103,30 @@ def main():
nodes_still_running = True
print('There are still instances running, waiting for them to be destroyed...')
# delete security groups, respecting dependencies (hence deleting 'control' and 'services' first)
# delete security groups
for group in conn.ex_list_security_groups():
# services depends on worker and api, so delete services first...
if group in ['services']:
print('Deleting security group: %s' % group)
conn.ex_delete_security_group(group)
for group in conn.ex_list_security_groups():
if group in ['control', 'services']:
# control depends on worker, so delete control before worker...
if group in ['control']:
print('Deleting security group: %s' % group)
conn.ex_delete_security_group(group)
# now we can delete security groups 'api' and 'worker', as 'control' and 'api' depended on them, otherwise AWS will
# throw DependencyViolation: resource has a dependent object
for group in conn.ex_list_security_groups():
if group in ['api', 'worker']:
if group in ['worker', 'api']:
print('Deleting security group: %s' % group)
conn.ex_delete_security_group(group)
# release elastic ips
for elastic_ip in conn.ex_describe_all_addresses():
if elastic_ip.instance_id is None:
print('Releasing unused elastic ip %s' % elastic_ip)
conn.ex_release_address(elastic_ip, domain=elastic_ip.domain)
if __name__ == '__main__':
main()

92
example-projects/demo4-scale-out-lb-in-aws.py
View File

@ -1,5 +1,6 @@
# import getpass
# import os
import configparser
from os.path import expanduser
# import libcloud.security
import time
@ -12,13 +13,16 @@ from libcloud.loadbalancer.base import Member, Algorithm
from libcloud.loadbalancer.types import Provider as loadbalancer_Provider
from libcloud.loadbalancer.providers import get_driver as loadbalancer_get_driver
# reqs:
# services: EC2 (nova, glance, neutron)
# resources: 2 instances, 2 elastic ips (1 keypair, 2 security groups)
home = expanduser("~")
# requirements:
# services: EC2, ELB
# resources: 2 instances, (1 keypair, 2 security groups),
# 1 (Classic) Elastic Load Balancer, expensive! delete it after you used it!
# The image to look for and use for the started instance
# ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20200408'
ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04 20200408 in us-west-1
ubuntu_image_name = 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20210128'
# TODO: 18.04, currently still needed for faafo, need to port faafo demo app to 20.04 or higher and python3...
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key
#
@ -26,7 +30,7 @@ ubuntu_image_id = "ami-085925f297f89fce1" # local ami id for resent ubuntu 18.04
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
keypair_name = 'srieger-pub'
pub_key_file = '~/.ssh/id_rsa.pub'
pub_key_file = home + '/.ssh/id_rsa.pub'
flavor_name = 't2.nano'
@ -34,11 +38,12 @@ flavor_name = 't2.nano'
# region_name = 'eu-central-1'
# region_name = 'ap-south-1'
# AWS Educate only allows us-east-1 see our AWS classroom at https://www.awseducate.com
# e.g., https://www.awseducate.com/student/s/launch-classroom?classroomId=a1v3m000005mNm6AAE
# AWS Academy Labs only allow us-east-1 see our AWS Academy Lab Guide, https://awsacademy.instructure.com/login/
region_name = 'us-east-1'
# starting instances in AWS Academy takes significantly longer compared to paid AWS accounts, allow ~ >2 minutes timeout
timeout = 600
def main():
###########################################################################
@ -47,17 +52,19 @@ def main():
#
###########################################################################
# see AWS Educate classroom, Account Details
# see AWS Academy Lab for Account Details
# read credentials from file
config = configparser.ConfigParser()
config.read_file(open(home + '/.aws/credentials'))
aws_access_key_id = config['default']['aws_access_key_id']
aws_secret_access_key = config['default']['aws_secret_access_key']
aws_session_token = config['default']['aws_session_token']
# hard coded AWS credentials using vars
# aws_access_key_id = "ASIAX..."
# aws_secret_access_key = "WLxxXK+..."
# aws_session_token = "FwoGZXIvYXdzEMb//////////wEaDE5rX.......0SleZ+L75I9iEri9LA4hovWul8HvexhCBK8.......................Ae/T+VkUbcQRtJEDwg+gYCABuk0JlSj5Wk7YA65r3BSNJXZFpkhbek6VBjvE/cEt5fKZEhENcdFxjAcAJLd6bOWi/oGXU5e3PX3mcXgm0oJpz6h3wqD1LvSDtw5GDwn0BHiF1Mu.......................cm/VukK5F"
# access_id = getpass.win_getpass("Enter your access_id:")
# secret_key = getpass.win_getpass("Enter your secret_key:")
# session_token = getpass.win_getpass("Enter your session_token:")
# access_id = "ASIAU..."
# secret_key = "7lafW..."
# session_token = "IQoJb3JpZ...EMb//..."
access_id = "ASIA..."
secret_key = "76lAj..."
session_token = "IQoJ..."
###########################################################################
#
@ -66,9 +73,9 @@ def main():
###########################################################################
provider = compute_get_driver(compute_Provider.EC2)
conn = provider(key=access_id,
secret=secret_key,
token=session_token,
conn = provider(key=aws_access_key_id,
secret=aws_secret_access_key,
token=aws_session_token,
region=region_name)
###########################################################################
@ -77,15 +84,32 @@ def main():
#
###########################################################################
print("Search for AMI...")
image = conn.list_images(ex_filters={"name": ubuntu_image_name})[0]
print("Using image: %s" % image)
# print("Fetching images (AMI) list from AWS region. This will take a lot of seconds (AWS has a very long list of "
# "supported operating systems and versions)... please be patient...")
# images = conn.list_images()
# image = ''
# for img in images:
# # if img.name == ubuntu_image_name:
# if img.extra['owner_alias'] == 'amazon':
# print(img)
# if img.id == ubuntu_image_name:
# image = img
# fetch/select the image referenced with ubuntu_image_name above
# image = [i for i in images if i.name == ubuntu_image_name][0]
# print(image)
# selecting the image based on defined AMI id
image = NodeImage(id=ubuntu_image_id, name=None, driver=conn)
# select image directly to save time, as retrieving the image list takes several minutes now,
# need to change ami id here if updated or for other regions, id is working for course in
# summer term 2022, in region: us-east-1 and pointing to ubuntu 18.04 used in the instance wizard,
# to update AMI id use the create instance wizard and copy amd64 image id for ubuntu 18.04 in the
# desired region
# image = NodeImage(id="ami-0e472ba40eb589f49",
# name=ubuntu_image_name,
# driver="hvm")
flavors = conn.list_sizes()
flavor = [s for s in flavors if s.id == flavor_name][0]
@ -132,7 +156,7 @@ def main():
print('Destroying Instance: %s' % instance.name)
conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups
# wait until all nodes are destroyed to be able to remove dependent security groups
nodes_still_running = True
while nodes_still_running:
nodes_still_running = False
@ -234,7 +258,7 @@ def main():
# Thanks to Stefan Friedmann for finding this fix ;)
userdata_service = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
-i database -i messaging
rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator
@ -258,7 +282,7 @@ def main():
###########################################################################
userdata_api = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
-i faafo -r api -m 'amqp://faafo:guest@%(services_ip)s:5672/' \
-d 'mysql+pymysql://faafo:password@%(services_ip)s:3306/faafo'
''' % {'services_ip': services_ip}
@ -291,7 +315,7 @@ def main():
###########################################################################
userdata_worker = '''#!/usr/bin/env bash
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh | bash -s -- \
curl -L -s https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install-aws.sh | bash -s -- \
-i faafo -r worker -e 'http://%(api_1_ip)s' -m 'amqp://faafo:guest@%(services_ip)s:5672/'
''' % {'api_1_ip': api_1_ip, 'services_ip': services_ip}
@ -335,9 +359,9 @@ def main():
###########################################################################
elb_provider = loadbalancer_get_driver(loadbalancer_Provider.ELB)
elb_conn = elb_provider(access_id,
secret_key,
token=session_token,
elb_conn = elb_provider(aws_access_key_id,
aws_secret_access_key,
token=aws_session_token,
region=region_name)
print("Deleting previously created load balancers in: " + str(elb_conn.list_balancers()))

Write Preview
Loading…
Cancel
Save