srieger
/
cloud-computing-msc-ai-examples
1
2
Fork 1
Code Issues Pull Requests 1 Releases Wiki Activity
Browse Source

updated demo1,2 and 3

pull/4/head
Sebastian Rieger 7 months ago
parent
5c06e7437b
commit
5e17f171ba
4 changed files with 0 additions and 733 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 126
      demo1-getting-started.py
  2. 235
      demo2-instance-with-init-script.py
  3. 275
      demo3-microservice.py
  4. 97
      destroy-all-demo-instances.py

126
demo1-getting-started.py
View File

@ -1,126 +0,0 @@
# Example for Cloud Computing Course Master AI / GSD
#
# uses libCloud: https://libcloud.apache.org/
# libCloud API documentation: https://libcloud.readthedocs.io/en/latest/
# OpenStack API documentation: https://developer.openstack.org/
# this code was initially based on the former tutorial: https://developer.openstack.org/firstapp-libcloud/
import getpass
from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions)
group_number = X
########################################################################################################################
#
# no changes necessary below this line in this example
#
########################################################################################################################
# web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
# your username in OpenStack
auth_username = 'CloudComp' + str(group_number)
# your project in OpenStack
project_name = 'CloudComp' + str(group_number)
# A network in the project the started instance will be attached to
project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance
#ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# default region
region_name = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username
# domain_name = "default"
def main():
# get the password from user
# auth_password = getpass.getpass("Enter your OpenStack password:")
auth_password = "demo"
# instantiate a connection to the OpenStack private cloud
# make sure to include ex_force_auth_version='3.x_password', as needed in our environment
provider = get_driver(Provider.OPENSTACK)
print(("Opening connection to %s as %s..." % (auth_url, auth_username)))
conn = provider(auth_username,
auth_password,
ex_force_auth_url=auth_url,
ex_force_auth_version='3.x_password',
ex_tenant_name=project_name,
ex_force_service_region=region_name)
# ex_domain_name=domain_name)
print("Getting images and selecting desired one...")
print("=========================================================================")
# get a list of images offered in the cloud context (e.g. Ubuntu 20.04, cirros, ...)
images = conn.list_images()
image = ''
for img in images:
if img.name == ubuntu_image_name:
image = img
print(img)
print("Getting flavors...")
print("=========================================================================")
# get a list of flavors offered in the cloud context (e.g. m1.small, m1.medium, ...)
flavors = conn.list_sizes()
for flavor in flavors:
print(flavor)
print("Selecting desired flavor...")
print("=========================================================================")
# get the flavor with id 2
flavor_id = '2'
flavor = conn.ex_get_size(flavor_id)
print(flavor)
print("Selecting desired network...")
print("=========================================================================")
# get a list of networks in the cloud context
networks = conn.ex_list_networks()
network = ''
for net in networks:
if net.name == project_network:
network = net
print("Create instance 'testing'...")
print("=========================================================================")
# create a new instance with the name "testing"
# make sure to provide networks (networks={network}) the instance should be attached to
instance_name = 'testing'
testing_instance = conn.create_node(name=instance_name, image=image, size=flavor, networks={network})
print(testing_instance)
print("Showing all running instances...")
print("=========================================================================")
# show all instances (running nodes) in the cloud context
instances = conn.list_nodes()
for instance in instances:
print(instance)
print("Destroying instance...")
print("=========================================================================")
# destroy the instance we have just created
conn.destroy_node(testing_instance)
# method that is called when the script is started from the command line
if __name__ == '__main__':
main()

235
demo2-instance-with-init-script.py
View File

@ -1,235 +0,0 @@
# import getpass
# import os
from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions)
group_number = X
# web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
# your username in OpenStack
auth_username = 'CloudComp' + str(group_number)
# your project in OpenStack
project_name = 'CloudComp' + str(group_number)
# A network in the project the started instance will be attached to
project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance
#ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key
#
# id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
#keypair_name = 'CloudComp30-keypair'
keypair_name = "srieger-pub"
pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small'
# default region
region_name = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username
# domain_name = "default"
def main():
###########################################################################
#
# get credentials
#
###########################################################################
# if "OS_PASSWORD" in os.environ:
# auth_password = os.environ["OS_PASSWORD"]
# else:
# auth_password = getpass.getpass("Enter your OpenStack password:")
auth_password = "demo"
###########################################################################
#
# create connection
#
###########################################################################
provider = get_driver(Provider.OPENSTACK)
conn = provider(auth_username,
auth_password,
ex_force_auth_url=auth_url,
ex_force_auth_version='3.x_password',
ex_tenant_name=project_name,
ex_force_service_region=region_name)
# ex_domain_name=domain_name)
###########################################################################
#
# get image, flavor, network for instance creation
#
###########################################################################
images = conn.list_images()
image = ''
for img in images:
if img.name == ubuntu_image_name:
image = img
flavors = conn.list_sizes()
flavor = ''
for flav in flavors:
if flav.name == flavor_name:
flavor = conn.ex_get_size(flav.id)
networks = conn.ex_list_networks()
network = ''
for net in networks:
if net.name == project_network:
network = net
###########################################################################
#
# create keypair dependency
#
###########################################################################
print('Checking for existing SSH key pair...')
keypair_exists = False
for keypair in conn.list_key_pairs():
if keypair.name == keypair_name:
keypair_exists = True
if keypair_exists:
print(('Keypair ' + keypair_name + ' already exists. Skipping import.'))
else:
print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file)
for keypair in conn.list_key_pairs():
print(keypair)
###########################################################################
#
# create security group dependency
#
###########################################################################
print('Checking for existing security group...')
security_group_name = 'all-in-one'
security_group_exists = False
all_in_one_security_group = ''
for security_group in conn.ex_list_security_groups():
if security_group.name == security_group_name:
all_in_one_security_group = security_group
security_group_exists = True
if security_group_exists:
print(('Security Group ' + all_in_one_security_group.name + ' already exists. Skipping creation.'))
else:
all_in_one_security_group = conn.ex_create_security_group(security_group_name,
'network access for all-in-one application.')
conn.ex_create_security_group_rule(all_in_one_security_group, 'TCP', 80, 80)
conn.ex_create_security_group_rule(all_in_one_security_group, 'TCP', 22, 22)
for security_group in conn.ex_list_security_groups():
print(security_group)
###########################################################################
#
# create all-in-one instance
#
###########################################################################
#hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh'
# testing / faafo dev branch:
hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh'
userdata = '''#!/usr/bin/env bash
curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i faafo -i messaging -r api -r worker -r demo
'''
print('Checking for existing instance...')
instance_name = 'all-in-one'
instance_exists = False
testing_instance = ''
for instance in conn.list_nodes():
if instance.name == instance_name:
testing_instance = instance
instance_exists = True
if instance_exists:
print(('Instance ' + testing_instance.name + ' already exists. Skipping creation.'))
exit()
else:
print('Starting new all-in-one instance and wait until it is running...')
testing_instance = conn.create_node(name=instance_name,
image=image,
size=flavor,
networks=[network],
ex_keyname=keypair_name,
ex_userdata=userdata,
ex_security_groups=[all_in_one_security_group])
conn.wait_until_running(nodes=[testing_instance], timeout=120, ssh_interface='private_ips')
###########################################################################
#
# assign all-in-one instance floating ip
#
###########################################################################
private_ip = None
if len(testing_instance.private_ips):
private_ip = testing_instance.private_ips[0]
print(('Private IP found: {}'.format(private_ip)))
public_ip = None
if len(testing_instance.public_ips):
public_ip = testing_instance.public_ips[0]
print(('Public IP found: {}'.format(public_ip)))
print('Checking for unused Floating IP...')
unused_floating_ip = None
for floating_ip in conn.ex_list_floating_ips():
if not floating_ip.node_id:
unused_floating_ip = floating_ip
break
if not unused_floating_ip and len(conn.ex_list_floating_ip_pools()):
pool = conn.ex_list_floating_ip_pools()[0]
print(('Allocating new Floating IP from pool: {}'.format(pool)))
unused_floating_ip = pool.create_floating_ip()
if public_ip:
print(('Instance ' + testing_instance.name + ' already has a public ip. Skipping attachment.'))
elif unused_floating_ip:
conn.ex_attach_floating_ip_to_node(testing_instance, unused_floating_ip)
actual_ip_address = None
if public_ip:
actual_ip_address = public_ip
elif unused_floating_ip:
actual_ip_address = unused_floating_ip.ip_address
elif private_ip:
actual_ip_address = private_ip
print('\n')
print(('The Fractals app will be deployed to http://{}\n'.format(actual_ip_address)))
print('You can use ssh to login to the instance using your private key. Default user name for official Ubuntu\n'
'Cloud Images is: ubuntu, so you can use, e.g.: "ssh -i ~/.ssh/id_rsa ubuntu@<floating-ip>" if your private\n'
'key is in the default location.\n\n'
'After login, you can list or "ssh ubuntu@<floating-ip>" available fractals using "faafo list". To request\n'
'the generation of new fractals, you can use "faafo create".\n\n'
'You can also see other options to use the faafo example cloud service using "faafo -h".\n\n'
'If you cannot start faafo command and/or do not see the webpage, you can check the Instance Console Log of\n'
'the instance, e.g., in OpenStack web interface.')
if __name__ == '__main__':
main()

275
demo3-microservice.py
View File

@ -1,275 +0,0 @@
# import getpass
# import os
from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider
# reqs:
# services: nova, glance, neutron
# resources: 2 instances, 2 floating ips (1 keypair, 2 security groups)
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions)
group_number = X
# web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
# your username in OpenStack
auth_username = 'CloudComp' + str(group_number)
# your project in OpenStack
project_name = 'CloudComp' + str(group_number)
# A network in the project the started instance will be attached to
project_network = 'CloudComp' + str(group_number) + '-net'
# The image to look for and use for the started instance
#ubuntu_image_name = "Ubuntu 20.04 - Focal Fossa - 64-bit - Cloud Based Image"
ubuntu_image_name = "Ubuntu 22.04 - Jammy Jellyfish - 64-bit - Cloud Based Image"
# The public key to be used for SSH connection, please make sure, that you have the corresponding private key
#
# id_rsa.pub should look like this (standard sshd pubkey format):
# ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAw+J...F3w2mleybgT1w== user@HOSTNAME
#keypair_name = 'CloudComp30-keypair'
keypair_name = "srieger-pub"
pub_key_file = '~/.ssh/id_rsa.pub'
flavor_name = 'm1.small'
# default region
region_name = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username
# domain_name = "default"
def main():
###########################################################################
#
# get credentials
#
###########################################################################
# if "OS_PASSWORD" in os.environ:
# auth_password = os.environ["OS_PASSWORD"]
# else:
# auth_password = getpass.getpass("Enter your OpenStack password:")
auth_password = "demo"
###########################################################################
#
# create connection
#
###########################################################################
provider = get_driver(Provider.OPENSTACK)
conn = provider(auth_username,
auth_password,
ex_force_auth_url=auth_url,
ex_force_auth_version='3.x_password',
ex_tenant_name=project_name,
ex_force_service_region=region_name)
# ex_domain_name=domain_name)
###########################################################################
#
# get image, flavor, network for instance creation
#
###########################################################################
images = conn.list_images()
image = ''
for img in images:
if img.name == ubuntu_image_name:
image = img
flavors = conn.list_sizes()
flavor = ''
for flav in flavors:
if flav.name == flavor_name:
flavor = conn.ex_get_size(flav.id)
networks = conn.ex_list_networks()
network = ''
for net in networks:
if net.name == project_network:
network = net
###########################################################################
#
# create keypair dependency
#
###########################################################################
print('Checking for existing SSH key pair...')
keypair_exists = False
for keypair in conn.list_key_pairs():
if keypair.name == keypair_name:
keypair_exists = True
if keypair_exists:
print(('Keypair ' + keypair_name + ' already exists. Skipping import.'))
else:
print('adding keypair...')
conn.import_key_pair_from_file(keypair_name, pub_key_file)
for keypair in conn.list_key_pairs():
print(keypair)
###########################################################################
#
# create security group dependency
#
###########################################################################
print('Checking for existing worker security group...')
security_group_name = 'worker'
security_group_exists = False
worker_security_group = ''
for security_group in conn.ex_list_security_groups():
if security_group.name == security_group_name:
worker_security_group = security_group
security_group_exists = True
if security_group_exists:
print(('Worker Security Group ' + worker_security_group.name + ' already exists. Skipping creation.'))
else:
worker_security_group = conn.ex_create_security_group('worker', 'for services that run on a worker node')
conn.ex_create_security_group_rule(worker_security_group, 'TCP', 22, 22)
print('Checking for existing controller security group...')
security_group_name = 'control'
security_group_exists = False
controller_security_group = ''
for security_group in conn.ex_list_security_groups():
if security_group.name == security_group_name:
controller_security_group = security_group
security_group_exists = True
if security_group_exists:
print(('Controller Security Group ' + controller_security_group.name + ' already exists. Skipping creation.'))
else:
controller_security_group = conn.ex_create_security_group('control', 'for services that run on a control node')
conn.ex_create_security_group_rule(controller_security_group, 'TCP', 22, 22)
conn.ex_create_security_group_rule(controller_security_group, 'TCP', 80, 80)
conn.ex_create_security_group_rule(controller_security_group, 'TCP', 5672, 5672,
source_security_group=worker_security_group)
for security_group in conn.ex_list_security_groups():
print(security_group)
###########################################################################
#
# create app-controller
#
###########################################################################
#hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/master/faafo/contrib/install.sh'
# testing / faafo dev branch:
hsfd_faafo_cloud_init_script = 'https://gogs.informatik.hs-fulda.de/srieger/cloud-computing-msc-ai-examples/raw/branch/dev_faafo/faafo/contrib/install.sh'
userdata = '''#!/usr/bin/env bash
curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i messaging -i faafo -r api
rabbitmqctl add_user faafo guest
rabbitmqctl set_user_tags faafo administrator
rabbitmqctl set_permissions -p / faafo ".*" ".*" ".*"
'''
print('Starting new app-controller instance and wait until it is running...')
instance_controller_1 = conn.create_node(name='app-controller',
image=image,
size=flavor,
networks=[network],
ex_keyname=keypair_name,
ex_userdata=userdata,
ex_security_groups=[controller_security_group])
conn.wait_until_running(nodes=[instance_controller_1], timeout=120, ssh_interface='private_ips')
###########################################################################
#
# assign app-controller floating ip
#
###########################################################################
print('Checking for unused Floating IP...')
unused_floating_ip = None
for floating_ip in conn.ex_list_floating_ips():
if not floating_ip.node_id:
unused_floating_ip = floating_ip
break
if not unused_floating_ip:
pool = conn.ex_list_floating_ip_pools()[0]
print(('Allocating new Floating IP from pool: {}'.format(pool)))
unused_floating_ip = pool.create_floating_ip()
conn.ex_attach_floating_ip_to_node(instance_controller_1, unused_floating_ip)
print(('Controller Application will be deployed to http://%s' % unused_floating_ip.ip_address))
###########################################################################
#
# getting id and ip address of app-controller instance
#
###########################################################################
# instance should not have a public ip? floating ips are assigned later
instance_controller_1 = conn.ex_get_node_details(instance_controller_1.id)
if instance_controller_1.public_ips:
ip_controller = instance_controller_1.public_ips[0]
else:
ip_controller = instance_controller_1.private_ips[0]
###########################################################################
#
# create app-worker-1
#
###########################################################################
userdata = '''#!/usr/bin/env bash
curl -L -s ''' + hsfd_faafo_cloud_init_script + ''' | bash -s -- \
-i faafo -r worker -e 'http://%(ip_controller)s' -m 'amqp://faafo:guest@%(ip_controller)s:5672/'
''' % {'ip_controller': ip_controller}
print('Starting new app-worker-1 instance and wait until it is running...')
instance_worker_1 = conn.create_node(name='app-worker-1',
image=image,
size=flavor,
networks=[network],
ex_keyname=keypair_name,
ex_userdata=userdata,
ex_security_groups=[worker_security_group])
conn.wait_until_running(nodes=[instance_worker_1], timeout=120, ssh_interface='private_ips')
###########################################################################
#
# assign app-worker floating ip
#
###########################################################################
print('Checking for unused Floating IP...')
unused_floating_ip = None
for floating_ip in conn.ex_list_floating_ips():
if not floating_ip.node_id:
unused_floating_ip = floating_ip
break
if not unused_floating_ip:
pool = conn.ex_list_floating_ip_pools()[0]
print(('Allocating new Floating IP from pool: {}'.format(pool)))
unused_floating_ip = pool.create_floating_ip()
conn.ex_attach_floating_ip_to_node(instance_worker_1, unused_floating_ip)
print(('The worker will be available for SSH at %s' % unused_floating_ip.ip_address))
print('You can use ssh to login to the controller using your private key. After login, you can list available '
'fractals using "faafo list". To request the generation of new fractals, you can use "faafo create". '
'You can also see other options to use the faafo example cloud service using "faafo -h".')
if __name__ == '__main__':
main()

97
destroy-all-demo-instances.py
View File

@ -1,97 +0,0 @@
# import getpass
# import os
# import libcloud.security
import time
from libcloud.compute.providers import get_driver
from libcloud.compute.types import Provider
# reqs:
# services: nova, glance, neutron
# resources: 2 instances (m1.small), 2 floating ips (1 keypair, 2 security groups)
# Please use 1-29 for X in the following variable to specify your group number. (will be used for the username,
# project etc., as coordinated in the lab sessions)
group_number = X
# web service endpoint of the private cloud infrastructure
auth_url = 'https://private-cloud.informatik.hs-fulda.de:5000'
# your username in OpenStack
auth_username = 'CloudComp' + str(group_number)
# your project in OpenStack
project_name = 'CloudComp' + str(group_number)
# default region
region_name = 'RegionOne'
# domain to use, "default" for local accounts, "hsfulda" for RZ LDAP, e.g., using fdaiXXXX as auth_username
domain_name = "default"
def main():
###########################################################################
#
# get credentials
#
###########################################################################
# if "OS_PASSWORD" in os.environ:
# auth_password = os.environ["OS_PASSWORD"]
# else:
# auth_password = getpass.getpass("Enter your OpenStack password:")
auth_password = "demo"
###########################################################################
#
# create connection
#
###########################################################################
# libcloud.security.VERIFY_SSL_CERT = False
provider = get_driver(Provider.OPENSTACK)
conn = provider(auth_username,
auth_password,
ex_force_auth_url=auth_url,
ex_force_auth_version='3.x_password',
ex_tenant_name=project_name,
ex_force_service_region=region_name,
ex_domain_name=domain_name)
###########################################################################
#
# clean up resources from previous demos
#
###########################################################################
# destroy running demo instances
for instance in conn.list_nodes():
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
'app-services', 'app-api-1', 'app-api-2']:
print(('Destroying Instance: %s' % instance.name))
conn.destroy_node(instance)
# wait until all nodes are destroyed to be able to remove depended security groups
nodes_still_running = True
while nodes_still_running:
nodes_still_running = False
time.sleep(3)
instances = conn.list_nodes()
for instance in instances:
# if we see any demo instances still running continue to wait for them to stop
if instance.name in ['all-in-one', 'app-worker-1', 'app-worker-2', 'app-worker-3', 'app-controller',
'app-services', 'app-api-1', 'app-api-2']:
nodes_still_running = True
print('There are still instances running, waiting for them to be destroyed...')
# delete security groups
for group in conn.ex_list_security_groups():
if group.name in ['control', 'worker', 'api', 'services']:
print(('Deleting security group: %s' % group.name))
conn.ex_delete_security_group(group)
if __name__ == '__main__':
main()
Write Preview
Loading…
Cancel
Save