srieger
/
cloud-computing-msc-ai-examples
1
2
Fork 1
Code Issues 1 Pull Requests Releases Wiki Activity
Browse Source

added example-projects

master
Sebastian Rieger 5 years ago
parent
ba80260324
commit
5b79d1fa19
10 changed files with 1157 additions and 1193 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      .idea/cloud-computing-msc-ai-examples.iml
  2. 7
      .idea/misc.xml
  3. 8
      .idea/modules.xml
  4. 7
      .idea/vcs.xml
  5. 241
      aws-turnserver/start.py
  6. 0
      example-projects/counter-demo/aws-cloudformation/cloudcomp-counter-demo-with-vpc-designer.png
  7. 1090
      example-projects/counter-demo/aws-cloudformation/cloudcomp-counter-demo-with-vpc.json
  8. 786
      example-projects/counter-demo/aws-cloudformation/cloudcomp-counter-demo.json
  9. 184
      example-projects/tug-of-war-in-the-clouds/aws-boto3-standalone-db/start.py
  10. 16
      example-projects/tug-of-war-in-the-clouds/aws-boto3-standalone-db/stop.py

11
.idea/cloud-computing-msc-ai-examples.iml
View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>

7
.idea/misc.xml
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/cloud-computing-msc-ai-examples.iml" filepath="$PROJECT_DIR$/.idea/cloud-computing-msc-ai-examples.iml" />
</modules>
</component>
</project>

7
.idea/vcs.xml
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
<mapping directory="$PROJECT_DIR$/aws-tug-of-war-in-the-clouds" vcs="Git" />
</component>
</project>

241
aws-turnserver/start.py
View File

@ -1,241 +0,0 @@
import boto3
from botocore.exceptions import ClientError
region = 'eu-central-1'
availabilityZone = 'eu-central-1b'
imageId = 'ami-0cc293023f983ed53'
instanceType = 't3.nano'
keyName = 'srieger-pub'
userData = ('#!/bin/bash\n'
'COTURN_VERSION="4.5.1.1"\n'
'LIBEVENT_VERSION="2.0.21"\n'
'\n'
'# extra repo for RedHat rpms\n'
'yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm\n'
'# essential tools\n'
'yum install -y joe htop git\n'
'# coturn requirements\n'
'yum install -y gcc openssl-devel\n'
'yum install -y sqlite-devel mysql-devel hiredis-devel mongo-c-driver-devel\n'
'\n'
'### libevent installation ###\n'
'wget https://github.com/downloads/libevent/libevent/libevent-$LIBEVENT_VERSION-stable.tar.gz\n'
'\n'
'tar xvfz libevent-$LIBEVENT_VERSION-stable.tar.gz\n'
'cd libevent-$LIBEVENT_VERSION-stable\n'
'./configure\n'
'make\n'
'make install\n'
'\n'
'### turnserver installation ###\n'
'wget https://coturn.net/turnserver/v$COTURN_VERSION/turnserver-$COTURN_VERSION.tar.gz\n'
'tar xvfz turnserver-$COTURN_VERSION.tar.gz\n'
'cd turnserver-$COTURN_VERSION\n'
'./configure\n'
'make\n'
'make install\n'
'\n'
'openssl req -new -subj "/CN=coturn" -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out /usr/local/etc/turn_server_cert.pem -keyout /usr/local/etc/turn_server_pkey.pem\n'
'\n'
'/usr/local/bin/turnadmin -a -u srieger -r hs-fulda.de -p coturnserver2019\n'
'/usr/local/bin/turnadmin -A -u srieger -p coturnserver2019\n'
'\n'
'MAC_ETH0=$(cat /sys/class/net/eth0/address)\n'
'MAC_ETH1=$(cat /sys/class/net/eth1/address)\n'
'LOCAL_IPV4S_ETH0=$(curl http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ETH0/local-ipv4s)\n'
'LOCAL_IPV4S_ETH1=$(curl http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ETH1/local-ipv4s)\n'
'PUBLIC_IPV4S_ETH0=$(curl http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ETH0/public-ipv4s)\n'
'PUBLIC_IPV4S_ETH1=$(curl http://169.254.169.254/latest/meta-data/network/interfaces/macs/$MAC_ETH1/public-ipv4s)\n'
'\n'
'cat <<EOF > /usr/local/etc/turnserver.conf\n'
'verbose\n'
'listening-ip=$LOCAL_IPV4S_ETH0\n'
'listening-ip=$LOCAL_IPV4S_ETH1\n'
'relay-ip=$LOCAL_IPV4S_ETH0\n'
'relay-ip=$LOCAL_IPV4S_ETH1\n'
'external-ip=$PUBLIC_IPV4S_ETH0/$LOCAL_IPV4S_ETH0\n'
'external-ip=$PUBLIC_IPV4S_ETH1/$LOCAL_IPV4S_ETH1\n'
'fingerprint\n'
'lt-cred-mech\n'
'#use-auth-secret\n'
'#static-auth-secret=751c45cae60a2839711a94c8d6bf0089e78b2149ca602fdXXXXXXXXXXXXXXXXX\n'
'realm=hs-fulda.de\n'
'total-quota=100\n'
'stale-nonce\n'
'cipher-list="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AES:RSA+3DES:!ADH:!AECDH:!MD5"\n'
'#no-stun\n'
'#no-loopback-peers\n'
'#no-multicast-peers\n'
'cli-password=coturnserver2019\n'
'web-admin\n'
'web-admin-ip=$LOCAL_IPV4S_ETH0\n'
'EOF\n'
'\n'
'/usr/local/bin/turnserver\n'
)
# convert with: cat install-coturn | sed "s/^/'/; s/$/\\\n'/"
client = boto3.setup_default_session(region_name=region)
ec2Client = boto3.client("ec2")
ec2Resource = boto3.resource('ec2')
response = ec2Client.describe_vpcs()
vpc_id = response.get('Vpcs', [{}])[0].get('VpcId', '')
subnet_id = ec2Client.describe_subnets(
Filters=[
{
'Name': 'availability-zone', 'Values': [availabilityZone]
}
])['Subnets'][0]['SubnetId']
print("Deleting old instance...")
print("------------------------------------")
response = ec2Client.describe_instances(Filters=[{'Name': 'tag-key', 'Values': ['coturn']}])
print(response)
reservations = response['Reservations']
for reservation in reservations:
for instance in reservation['Instances']:
if instance['State']['Name'] == "running" or instance['State']['Name'] == "pending":
response = ec2Client.terminate_instances(InstanceIds=[instance['InstanceId']])
print(response)
instanceToTerminate = ec2Resource.Instance(instance['InstanceId'])
instanceToTerminate.wait_until_terminated()
print("Delete old security group...")
print("------------------------------------")
try:
response = ec2Client.delete_security_group(GroupName='coturn')
except ClientError as e:
print(e)
print("Delete old elastic ips...")
print("------------------------------------")
try:
response = ec2Client.describe_addresses(Filters=[{'Name': 'tag-key', 'Values': ['coturn']}])
addresses = response['Addresses']
for address in addresses:
ec2Client.release_address(AllocationId=address['AllocationId'])
except ClientError as e:
print(e)
print("Create security group...")
print("------------------------------------")
try:
response = ec2Client.create_security_group(GroupName='coturn',
Description='coturn',
VpcId=vpc_id)
security_group_id = response['GroupId']
print('Security Group Created %s in vpc %s.' % (security_group_id, vpc_id))
data = ec2Client.authorize_security_group_ingress(
GroupId=security_group_id,
IpPermissions=[
{'IpProtocol': 'tcp',
'FromPort': 3478,
'ToPort': 3478,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'udp',
'FromPort': 3478,
'ToPort': 3478,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'tcp',
'FromPort': 5349,
'ToPort': 5349,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'udp',
'FromPort': 5349,
'ToPort': 5349,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'tcp',
'FromPort': 8080,
'ToPort': 8080,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'udp',
'FromPort': 49152,
'ToPort': 65535,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'tcp',
'FromPort': 22,
'ToPort': 22,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]}
])
print('Ingress Successfully Set %s' % data)
except ClientError as e:
print(e)
print("Allocate additional elastic ips...")
print("------------------------------------")
response = ec2Client.allocate_address(
Domain='vpc',
)
firstIpAddressAllocationId = response['AllocationId']
ec2Client.create_tags(Resources=[firstIpAddressAllocationId], Tags=[{'Key': 'coturn', 'Value': 'installed'}])
response = ec2Client.allocate_address(
Domain='vpc',
)
secondIpAddressAllocationId = response['AllocationId']
ec2Client.create_tags(Resources=[secondIpAddressAllocationId], Tags=[{'Key': 'coturn', 'Value': 'installed'}])
print("Running new instance...")
print("------------------------------------")
response = ec2Client.run_instances(
ImageId=imageId,
InstanceType=instanceType,
Placement={'AvailabilityZone': availabilityZone, },
KeyName=keyName,
MinCount=1,
MaxCount=1,
UserData=userData,
NetworkInterfaces=[
{
'DeviceIndex': 0,
'Groups': [
security_group_id,
],
'SubnetId': subnet_id,
},
{
'DeviceIndex': 1,
'Groups': [
security_group_id,
],
'SubnetId': subnet_id,
},
],
TagSpecifications=[
{
'ResourceType': 'instance',
'Tags': [
{'Key': 'coturn', 'Value': 'installed'}
],
}
],
)
instanceId = response['Instances'][0]['InstanceId']
firstNetworkInterfaceId = response['Instances'][0]['NetworkInterfaces'][0]['NetworkInterfaceId']
secondNetworkInterfaceId = response['Instances'][0]['NetworkInterfaces'][1]['NetworkInterfaceId']
instance = ec2Resource.Instance(instanceId)
instance.wait_until_running()
response = ec2Client.associate_address(
AllocationId=firstIpAddressAllocationId,
NetworkInterfaceId=firstNetworkInterfaceId,
)
response = ec2Client.associate_address(
AllocationId=secondIpAddressAllocationId,
NetworkInterfaceId=secondNetworkInterfaceId,
)
print(instanceId)

0
aws-cloudformation-demo/cloudcomp-counter-demo-with-vpc-designer.png → example-projects/counter-demo/aws-cloudformation/cloudcomp-counter-demo-with-vpc-designer.png
View File

Side by Side Swipe Overlay

Before

Width: 2046  |  Height: 2046  |  Size: 299 KiB

After

Width: 2046  |  Height: 2046  |  Size: 299 KiB

1090
example-projects/counter-demo/aws-cloudformation/cloudcomp-counter-demo-with-vpc.json
File diff suppressed because it is too large
View File

786
aws-cloudformation-demo/cloudcomp-counter-demo.json → example-projects/counter-demo/aws-cloudformation/cloudcomp-counter-demo.json
View File

@ -1,394 +1,394 @@
{
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"paramImageID": {
"Type": "AWS::EC2::Image::Id",
"Default": "ami-f573e19a",
"Description": "Amazon Image ID"
},
"paramInstanceType": {
"Type": "String",
"Default": "t2.nano",
"AllowedValues": [
"t2.nano",
"t2.micro",
"m1.small"
],
"Description": "Enter t2.micro, m1.small, or m1.large. Default is t2.micro."
},
"paramKeyPair": {
"Type": "AWS::EC2::KeyPair::KeyName",
"Description": "Amazon EC2 Key Pair"
},
"paramIamInstanceRole": {
"Type": "String",
"Default": "ec2-s3-vertsys-access-role",
"Description": "IAM Instance Role"
},
"paramVPC": {
"Type": "AWS::EC2::VPC::Id",
"Description": "VPC"
},
"paramSubnetIDs": {
"Type": "List<AWS::EC2::Subnet::Id>",
"Description": "Subnet IDs"
},
"paramAvailabilityZones": {
"Type": "List<AWS::EC2::AvailabilityZone::Name>",
"Description": "AvailabilityZones"
}
},
"Resources": {
"lcVertSysAutoScaleConfigv11": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"AssociatePublicIpAddress": true,
"ImageId": {
"Ref": "paramImageID"
},
"InstanceType": {
"Ref": "paramInstanceType"
},
"KeyName": {
"Ref": "paramKeyPair"
},
"IamInstanceProfile": {
"Ref": "paramIamInstanceRole"
},
"SecurityGroups": [
{
"Ref": "sgCloudCompDemoSecurityGroup"
}
],
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash\n",
"\n",
"# this script will be run during the boot process by each VertSys instance created\n",
"# in AWS currently this is a plain bash script that requires a RedHat based image\n",
"# (AMI) could be ported to cloud-init for better compatibility with other Linux\n",
"# distros\n",
"#\n",
"# see https://docs.aws.amazon.com/de_de/AWSEC2/latest/UserGuide/user-data.html\n",
"\n",
"# Config\n",
"SCRIPT_ROOT_PATH=\"/tmp/init-script\"\n",
"VERTSYS_PATH=\"$SCRIPT_ROOT_PATH/verteilte-systeme-bsc-ai-examples/VerteilteSysteme-Examples/build/\"\n",
"#JARS = \"TCPServer.jar TCPServerMulti.jar UDPServer.jar UDPServerMulti.jar UDPTimeCounterServer.jar TCPTimeCounterServer.jar TCPPerfServer.jar\"\n",
"JARS=\"TCPServer.jar TCPPerfServer.jar UDPServer.jar UDPTimeCounterServer.jar TCPTimeCounterServer.jar TCPTimeCounterRESTServer.jar\"\n",
"REPO=\"https://gogs.informatik.hs-fulda.de/srieger/verteilte-systeme-bsc-ai-examples.git\"\n",
"\n",
"# Create path to run the script\n",
"mkdir $SCRIPT_ROOT_PATH\n",
"cd $SCRIPT_ROOT_PATH\n",
"\n",
"# fetch user-data (should be the content of this script itself ;)) to be able check it later and run it again\n",
"wget http://169.254.169.254/latest/user-data\n",
"chmod +x user-data\n",
"wget http://169.254.169.254/latest/meta-data/placement/availability-zone\n",
"\n",
"# setting console prompt to include location (availability zone and region in AWS)\n",
"echo \"PS1='[\\u@\\h@\\e[32m`cat /tmp/init-script/availability-zone`\\e[32m \\W]$ '\" >>/root/.bashrc\n",
"echo \"PS1='[\\u@\\h@\\e[32m`cat /tmp/init-script/availability-zone`\\e[39m \\W]$ '\" >>/home/ec2-user/.bashrc\n",
"\n",
"# wait for internet connection, should not be necessary, but just in case ;)\n",
"while ! ping -c 1 -W 1 8.8.8.8; do\n",
" echo \"Waiting for 8.8.8.8 - network interface might be down...\"\n",
" sleep 1\n",
"done\n",
"\n",
"# installation of required packages\n",
"echo \"Installing packages...\"\n",
"sudo yum update -y\n",
"sudo yum install -y java git\n",
"#rm -rf verteilte-systeme-bsc-ai-examples\n",
"if [ ! -d $VERTSYS_PATH ]; then\n",
" echo \"Cloning repo...\"\n",
" git clone $REPO\n",
"fi\n",
"\n",
"# killall running screens, .e.g to cleanup if script is run again after boot\n",
"killall screen\n",
"# start all jars\n",
"echo \"Starting JARS: ${JARS}...\"\n",
"for JAR in $JARS; do\n",
" echo \"Starting ${JAR}...\"\n",
" sudo screen -dmS $JAR -L java -jar $VERTSYS_PATH/$JAR\n",
"done\n",
"\n",
"# wait a second, to allow java services to start\n",
"sleep 1\n",
"\n",
"# output status of ports 36000-36199 and all running processes\n",
"echo \"Status:\"\n",
"sudo netstat -taupen | grep 36[0,1][0-9][0-9]\n",
"sudo ps aux | grep java\n",
"\n",
"# Example for path and contents:\n",
"#\n",
"# ./verteilte-systeme-bsc-ai-examples/VerteilteSysteme-Examples/build/\n",
"#\n",
"#build-server-jars.xml TCPServerMulti.jar\n",
"#RMIEchoServer.jar UDPServer.jar\n",
"#RMIMandelbrotCalculationsServer.jar UDPServerMulti.jar\n",
"#TCPPerfServer.jar UDPTimeCounterServer.jar\n",
"#TCPServer.jar\n"
]
]
}
}
}
},
"sgCloudCompDemoSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "CloudComp Counter Demo",
"VpcId": {
"Ref": "paramVPC"
}
}
},
"ingress1": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "tcp",
"FromPort": "36037",
"ToPort": "36137",
"CidrIp": "0.0.0.0/0"
}
},
"ingress2": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": "0.0.0.0/0"
}
},
"ingress3": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "udp",
"FromPort": "36037",
"ToPort": "36137",
"CidrIp": "0.0.0.0/0"
}
},
"egress1": {
"Type": "AWS::EC2::SecurityGroupEgress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "-1",
"CidrIp": "0.0.0.0/0"
}
},
"asgCloudCompAutoScaleGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"AvailabilityZones": {
"Ref": "paramAvailabilityZones"
},
"TargetGroupARNs": [
{
"Ref": "elbCloudCompTargetGroup"
}
],
"Cooldown": "30",
"DesiredCapacity": "1",
"HealthCheckGracePeriod": "60",
"HealthCheckType": "EC2",
"MaxSize": "3",
"MinSize": "1",
"VPCZoneIdentifier": {
"Ref": "paramSubnetIDs"
},
"LaunchConfigurationName": {
"Ref": "lcVertSysAutoScaleConfigv11"
},
"MetricsCollection": [
{
"Granularity": "1Minute",
"Metrics": [
"GroupPendingInstances",
"GroupMinSize",
"GroupDesiredCapacity",
"GroupTerminatingInstances",
"GroupInServiceInstances",
"GroupStandbyInstances",
"GroupMaxSize",
"GroupTotalInstances"
]
}
],
"TerminationPolicies": [
"Default"
]
}
},
"sgCloudCompScalePolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"DependsOn": "elbCloudCompListener",
"Properties": {
"PolicyType": "TargetTrackingScaling",
"EstimatedInstanceWarmup": 60,
"TargetTrackingConfiguration": {
"DisableScaleIn": false,
"TargetValue": 5,
"PredefinedMetricSpecification": {
"PredefinedMetricType": "ALBRequestCountPerTarget",
"ResourceLabel": {
"Fn::Join": [
"/",
[
"app/elbCloudCompLoadBalancer",
{
"Fn::Select": [
"3",
{
"Fn::Split": [
"/",
{
"Ref": "elbCloudCompLoadBalancer"
}
]
}
]
},
"targetgroup/elbCloudCompTargetGroup",
{
"Fn::Select": [
"2",
{
"Fn::Split": [
"/",
{
"Ref": "elbCloudCompTargetGroup"
}
]
}
]
}
]
]
}
}
},
"AutoScalingGroupName": {
"Ref": "asgCloudCompAutoScaleGroup"
}
}
},
"elbCloudCompLoadBalancer": {
"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties": {
"Name": "elbCloudCompLoadBalancer",
"IpAddressType": "ipv4",
"Type": "application",
"Scheme": "internet-facing",
"SecurityGroups": [
{
"Ref": "sgCloudCompDemoSecurityGroup"
}
],
"Subnets": {
"Ref": "paramSubnetIDs"
}
}
},
"elbCloudCompTargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Port": 36042,
"HealthCheckIntervalSeconds": 30,
"HealthCheckTimeoutSeconds": 5,
"HealthyThresholdCount": 2,
"UnhealthyThresholdCount": 2,
"HealthCheckPath": "/counter",
"HealthCheckProtocol": "HTTP",
"TargetGroupAttributes": [
{
"Key": "deregistration_delay.timeout_seconds",
"Value": "20"
}
],
"Protocol": "HTTP",
"TargetType": "instance",
"Matcher": {
"HttpCode": "200"
},
"Name": "elbCloudCompTargetGroup",
"VpcId": {
"Ref": "paramVPC"
}
}
},
"elbCloudCompListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"LoadBalancerArn": {
"Ref": "elbCloudCompLoadBalancer"
},
"Protocol": "HTTP",
"Port": 36042,
"DefaultActions": [
{
"Type": "forward",
"TargetGroupArn": {
"Ref": "elbCloudCompTargetGroup"
}
}
]
}
}
},
"Outputs": {
"LoadBalancer": {
"Description": "Load Balancer",
"Value": {
"Ref": "elbCloudCompLoadBalancer"
}
},
"LoadBalancerDns": {
"Description": "Load Balancer DNS",
"Value": {
"Fn::GetAtt": [
"elbCloudCompLoadBalancer",
"DNSName"
]
}
},
"LoadBalancerURL": {
"Description": "Load Balancer URL",
"Value": {
"Fn::Join": [
":",
[
{
"Fn::GetAtt": [
"elbCloudCompLoadBalancer",
"DNSName"
]
},
"36042/counter"
]
]
}
}
},
"Description": "CloudComp Counter Demo"
{
"AWSTemplateFormatVersion": "2010-09-09",
"Parameters": {
"paramImageID": {
"Type": "AWS::EC2::Image::Id",
"Default": "ami-f573e19a",
"Description": "Amazon Image ID"
},
"paramInstanceType": {
"Type": "String",
"Default": "t2.nano",
"AllowedValues": [
"t2.nano",
"t2.micro",
"m1.small"
],
"Description": "Enter t2.micro, m1.small, or m1.large. Default is t2.micro."
},
"paramKeyPair": {
"Type": "AWS::EC2::KeyPair::KeyName",
"Description": "Amazon EC2 Key Pair"
},
"paramIamInstanceRole": {
"Type": "String",
"Default": "ec2-s3-vertsys-access-role",
"Description": "IAM Instance Role"
},
"paramVPC": {
"Type": "AWS::EC2::VPC::Id",
"Description": "VPC"
},
"paramSubnetIDs": {
"Type": "List<AWS::EC2::Subnet::Id>",
"Description": "Subnet IDs"
},
"paramAvailabilityZones": {
"Type": "List<AWS::EC2::AvailabilityZone::Name>",
"Description": "AvailabilityZones"
}
},
"Resources": {
"lcVertSysAutoScaleConfigv11": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"AssociatePublicIpAddress": true,
"ImageId": {
"Ref": "paramImageID"
},
"InstanceType": {
"Ref": "paramInstanceType"
},
"KeyName": {
"Ref": "paramKeyPair"
},
"IamInstanceProfile": {
"Ref": "paramIamInstanceRole"
},
"SecurityGroups": [
{
"Ref": "sgCloudCompDemoSecurityGroup"
}
],
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash\n",
"\n",
"# this script will be run during the boot process by each VertSys instance created\n",
"# in AWS currently this is a plain bash script that requires a RedHat based image\n",
"# (AMI) could be ported to cloud-init for better compatibility with other Linux\n",
"# distros\n",
"#\n",
"# see https://docs.aws.amazon.com/de_de/AWSEC2/latest/UserGuide/user-data.html\n",
"\n",
"# Config\n",
"SCRIPT_ROOT_PATH=\"/tmp/init-script\"\n",
"VERTSYS_PATH=\"$SCRIPT_ROOT_PATH/verteilte-systeme-bsc-ai-examples/VerteilteSysteme-Examples/build/\"\n",
"#JARS = \"TCPServer.jar TCPServerMulti.jar UDPServer.jar UDPServerMulti.jar UDPTimeCounterServer.jar TCPTimeCounterServer.jar TCPPerfServer.jar\"\n",
"JARS=\"TCPServer.jar TCPPerfServer.jar UDPServer.jar UDPTimeCounterServer.jar TCPTimeCounterServer.jar TCPTimeCounterRESTServer.jar\"\n",
"REPO=\"https://gogs.informatik.hs-fulda.de/srieger/verteilte-systeme-bsc-ai-examples.git\"\n",
"\n",
"# Create path to run the script\n",
"mkdir $SCRIPT_ROOT_PATH\n",
"cd $SCRIPT_ROOT_PATH\n",
"\n",
"# fetch user-data (should be the content of this script itself ;)) to be able check it later and run it again\n",
"wget http://169.254.169.254/latest/user-data\n",
"chmod +x user-data\n",
"wget http://169.254.169.254/latest/meta-data/placement/availability-zone\n",
"\n",
"# setting console prompt to include location (availability zone and region in AWS)\n",
"echo \"PS1='[\\u@\\h@\\e[32m`cat /tmp/init-script/availability-zone`\\e[32m \\W]$ '\" >>/root/.bashrc\n",
"echo \"PS1='[\\u@\\h@\\e[32m`cat /tmp/init-script/availability-zone`\\e[39m \\W]$ '\" >>/home/ec2-user/.bashrc\n",
"\n",
"# wait for internet connection, should not be necessary, but just in case ;)\n",
"while ! ping -c 1 -W 1 8.8.8.8; do\n",
" echo \"Waiting for 8.8.8.8 - network interface might be down...\"\n",
" sleep 1\n",
"done\n",
"\n",
"# installation of required packages\n",
"echo \"Installing packages...\"\n",
"sudo yum update -y\n",
"sudo yum install -y java git\n",
"#rm -rf verteilte-systeme-bsc-ai-examples\n",
"if [ ! -d $VERTSYS_PATH ]; then\n",
" echo \"Cloning repo...\"\n",
" git clone $REPO\n",
"fi\n",
"\n",
"# killall running screens, .e.g to cleanup if script is run again after boot\n",
"killall screen\n",
"# start all jars\n",
"echo \"Starting JARS: ${JARS}...\"\n",
"for JAR in $JARS; do\n",
" echo \"Starting ${JAR}...\"\n",
" sudo screen -dmS $JAR -L java -jar $VERTSYS_PATH/$JAR\n",
"done\n",
"\n",
"# wait a second, to allow java services to start\n",
"sleep 1\n",
"\n",
"# output status of ports 36000-36199 and all running processes\n",
"echo \"Status:\"\n",
"sudo netstat -taupen | grep 36[0,1][0-9][0-9]\n",
"sudo ps aux | grep java\n",
"\n",
"# Example for path and contents:\n",
"#\n",
"# ./verteilte-systeme-bsc-ai-examples/VerteilteSysteme-Examples/build/\n",
"#\n",
"#build-server-jars.xml TCPServerMulti.jar\n",
"#RMIEchoServer.jar UDPServer.jar\n",
"#RMIMandelbrotCalculationsServer.jar UDPServerMulti.jar\n",
"#TCPPerfServer.jar UDPTimeCounterServer.jar\n",
"#TCPServer.jar\n"
]
]
}
}
}
},
"sgCloudCompDemoSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "CloudComp Counter Demo",
"VpcId": {
"Ref": "paramVPC"
}
}
},
"ingress1": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "tcp",
"FromPort": "36037",
"ToPort": "36137",
"CidrIp": "0.0.0.0/0"
}
},
"ingress2": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": "0.0.0.0/0"
}
},
"ingress3": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "udp",
"FromPort": "36037",
"ToPort": "36137",
"CidrIp": "0.0.0.0/0"
}
},
"egress1": {
"Type": "AWS::EC2::SecurityGroupEgress",
"Properties": {
"GroupId": {
"Ref": "sgCloudCompDemoSecurityGroup"
},
"IpProtocol": "-1",
"CidrIp": "0.0.0.0/0"
}
},
"asgCloudCompAutoScaleGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"AvailabilityZones": {
"Ref": "paramAvailabilityZones"
},
"TargetGroupARNs": [
{
"Ref": "elbCloudCompTargetGroup"
}
],
"Cooldown": "30",
"DesiredCapacity": "1",
"HealthCheckGracePeriod": "60",
"HealthCheckType": "EC2",
"MaxSize": "3",
"MinSize": "1",
"VPCZoneIdentifier": {
"Ref": "paramSubnetIDs"
},
"LaunchConfigurationName": {
"Ref": "lcVertSysAutoScaleConfigv11"
},
"MetricsCollection": [
{
"Granularity": "1Minute",
"Metrics": [
"GroupPendingInstances",
"GroupMinSize",
"GroupDesiredCapacity",
"GroupTerminatingInstances",
"GroupInServiceInstances",
"GroupStandbyInstances",
"GroupMaxSize",
"GroupTotalInstances"
]
}
],
"TerminationPolicies": [
"Default"
]
}
},
"sgCloudCompScalePolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"DependsOn": "elbCloudCompListener",
"Properties": {
"PolicyType": "TargetTrackingScaling",
"EstimatedInstanceWarmup": 60,
"TargetTrackingConfiguration": {
"DisableScaleIn": false,
"TargetValue": 5,
"PredefinedMetricSpecification": {
"PredefinedMetricType": "ALBRequestCountPerTarget",
"ResourceLabel": {
"Fn::Join": [
"/",
[
"app/elbCloudCompLoadBalancer",
{
"Fn::Select": [
"3",
{
"Fn::Split": [
"/",
{
"Ref": "elbCloudCompLoadBalancer"
}
]
}
]
},
"targetgroup/elbCloudCompTargetGroup",
{
"Fn::Select": [
"2",
{
"Fn::Split": [
"/",
{
"Ref": "elbCloudCompTargetGroup"
}
]
}
]
}
]
]
}
}
},
"AutoScalingGroupName": {
"Ref": "asgCloudCompAutoScaleGroup"
}
}
},
"elbCloudCompLoadBalancer": {
"Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties": {
"Name": "elbCloudCompLoadBalancer",
"IpAddressType": "ipv4",
"Type": "application",
"Scheme": "internet-facing",
"SecurityGroups": [
{
"Ref": "sgCloudCompDemoSecurityGroup"
}
],
"Subnets": {
"Ref": "paramSubnetIDs"
}
}
},
"elbCloudCompTargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Port": 36042,
"HealthCheckIntervalSeconds": 30,
"HealthCheckTimeoutSeconds": 5,
"HealthyThresholdCount": 2,
"UnhealthyThresholdCount": 2,
"HealthCheckPath": "/counter",
"HealthCheckProtocol": "HTTP",
"TargetGroupAttributes": [
{
"Key": "deregistration_delay.timeout_seconds",
"Value": "20"
}
],
"Protocol": "HTTP",
"TargetType": "instance",
"Matcher": {
"HttpCode": "200"
},
"Name": "elbCloudCompTargetGroup",
"VpcId": {
"Ref": "paramVPC"
}
}
},
"elbCloudCompListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"LoadBalancerArn": {
"Ref": "elbCloudCompLoadBalancer"
},
"Protocol": "HTTP",
"Port": 36042,
"DefaultActions": [
{
"Type": "forward",
"TargetGroupArn": {
"Ref": "elbCloudCompTargetGroup"
}
}
]
}
}
},
"Outputs": {
"LoadBalancer": {
"Description": "Load Balancer",
"Value": {
"Ref": "elbCloudCompLoadBalancer"
}
},
"LoadBalancerDns": {
"Description": "Load Balancer DNS",
"Value": {
"Fn::GetAtt": [
"elbCloudCompLoadBalancer",
"DNSName"
]
}
},
"LoadBalancerURL": {
"Description": "Load Balancer URL",
"Value": {
"Fn::Join": [
":",
[
{
"Fn::GetAtt": [
"elbCloudCompLoadBalancer",
"DNSName"
]
},
"36042/counter"
]
]
}
}
},
"Description": "CloudComp Counter Demo"
} }

184
example-projects/tug-of-war-in-the-clouds/aws-boto3-standalone-db/start.py
View File

@ -0,0 +1,184 @@
import boto3
from botocore.exceptions import ClientError
region = 'eu-central-1'
availabilityZone = 'eu-central-1b'
imageId = 'ami-0cc293023f983ed53'
instanceType = 't3.nano'
keyName = 'srieger-pub'
userDataDB = ('#!/bin/bash\n'
'#!/bin/bash\n'
'# extra repo for RedHat rpms\n'
'yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm\n'
'# essential tools\n'
'yum install -y joe htop git\n'
'# mysql\n'
'yum install -y mariadb mariadb-server\n'
'\n'
'service mariadb start\n'
'\n'
'echo "create database cloud_tug_of_war" | mysql -u root\n'
'\n'
'echo "create table clouds ( cloud_id INT AUTO_INCREMENT, name VARCHAR(255) NOT NULL, value INT, max_value INT, PRIMARY KEY (cloud_id))" | mysql -u root cloud_tug_of_war\n'
'\n'
'echo "CREATE USER \'cloud_tug_of_war\'@\'%\' IDENTIFIED BY \'cloud\';" | mysql -u root\n'
'echo "GRANT ALL PRIVILEGES ON cloud_tug_of_war.* TO \'cloud_tug_of_war\'@\'%\';" | mysql -u root\n'
'echo "FLUSH PRIVILEGES" | mysql -u root\n'
)
# convert with: cat install-mysql | sed "s/^/'/; s/$/\\\n'/"
client = boto3.setup_default_session(region_name=region)
ec2Client = boto3.client("ec2")
ec2Resource = boto3.resource('ec2')
response = ec2Client.describe_vpcs()
vpc_id = response.get('Vpcs', [{}])[0].get('VpcId', '')
subnet_id = ec2Client.describe_subnets(
Filters=[
{
'Name': 'availability-zone', 'Values': [availabilityZone]
}
])['Subnets'][0]['SubnetId']
print("Deleting old instance...")
print("------------------------------------")
response = ec2Client.describe_instances(Filters=[{'Name': 'tag-key', 'Values': ['tug-of-war']}])
print(response)
reservations = response['Reservations']
for reservation in reservations:
for instance in reservation['Instances']:
if instance['State']['Name'] == "running" or instance['State']['Name'] == "pending":
response = ec2Client.terminate_instances(InstanceIds=[instance['InstanceId']])
print(response)
instanceToTerminate = ec2Resource.Instance(instance['InstanceId'])
instanceToTerminate.wait_until_terminated()
print("Delete old security group...")
print("------------------------------------")
try:
response = ec2Client.delete_security_group(GroupName='tug-of-war')
except ClientError as e:
print(e)
print("Create security group...")
print("------------------------------------")
try:
response = ec2Client.create_security_group(GroupName='tug-of-war',
Description='tug-of-war',
VpcId=vpc_id)
security_group_id = response['GroupId']
print('Security Group Created %s in vpc %s.' % (security_group_id, vpc_id))
data = ec2Client.authorize_security_group_ingress(
GroupId=security_group_id,
IpPermissions=[
{'IpProtocol': 'tcp',
'FromPort': 3306,
'ToPort': 3306,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'tcp',
'FromPort': 22,
'ToPort': 22,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'tcp',
'FromPort': 80,
'ToPort': 80,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]},
{'IpProtocol': 'tcp',
'FromPort': 443,
'ToPort': 443,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}]}
])
print('Ingress Successfully Set %s' % data)
except ClientError as e:
print(e)
print("Running new DB instance...")
print("------------------------------------")
response = ec2Client.run_instances(
ImageId=imageId,
InstanceType=instanceType,
Placement={'AvailabilityZone': availabilityZone, },
KeyName=keyName,
MinCount=1,
MaxCount=1,
UserData=userDataDB,
SecurityGroupIds=[
security_group_id,
],
TagSpecifications=[
{
'ResourceType': 'instance',
'Tags': [
{'Key': 'Name', 'Value': 'tug-of-war-db1'},
{'Key': 'tug-of-war', 'Value': 'installed'}
],
}
],
)
instanceIdDB = response['Instances'][0]['InstanceId']
privateIpDB = response['Instances'][0]['PrivateIpAddress']
#privateIpDB = response['Instances'][0]['NetworkInterfaces'][0]['NetworkInterfaceId']
instance = ec2Resource.Instance(instanceIdDB)
instance.wait_until_running()
print(instanceIdDB)
userDataWebServer = ('#!/bin/bash\n'
'# extra repo for RedHat rpms\n'
'yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm\n'
'# essential tools\n'
'yum install -y joe htop git\n'
'# mysql\n'
'yum install -y httpd php php-mysql\n'
'\n'
'service httpd start\n'
'\n'
'wget http://mmnet.informatik.hs-fulda.de/cloudcomp/tug-of-war-in-the-clouds.tar.gz\n'
'cp tug-of-war-in-the-clouds.tar.gz /var/www/html/\n'
'cd /var/www/html\n'
'tar zxvf tug-of-war-in-the-clouds.tar.gz\n'
'\n'
'# change hostname of db connection\n'
'sed -i s/localhost/' + privateIpDB + '/g /var/www/html/config.php\n'
)
print("Running new Web Server instance...")
print("------------------------------------")
response = ec2Client.run_instances(
ImageId=imageId,
InstanceType=instanceType,
Placement={'AvailabilityZone': availabilityZone, },
KeyName=keyName,
MinCount=1,
MaxCount=1,
UserData=userDataWebServer,
SecurityGroupIds=[
security_group_id,
],
TagSpecifications=[
{
'ResourceType': 'instance',
'Tags': [
{'Key': 'Name', 'Value': 'tug-of-war-webserver1'},
{'Key': 'tug-of-war', 'Value': 'installed'}
],
}
],
)
instanceIdDB = response['Instances'][0]['InstanceId']
privateIpDB = response['Instances'][0]['PrivateIpAddress']
#privateIpDB = response['Instances'][0]['NetworkInterfaces'][0]['NetworkInterfaceId']
instance = ec2Resource.Instance(instanceIdDB)
instance.wait_until_running()

16
aws-turnserver/stop.py → example-projects/tug-of-war-in-the-clouds/aws-boto3-standalone-db/stop.py
View File

@ -13,7 +13,7 @@ vpc_id = response.get('Vpcs', [{}])[0].get('VpcId', '')
print("Deleting old instance...") print("Deleting old instance...")
print("------------------------------------") print("------------------------------------")
response = ec2Client.describe_instances(Filters=[{'Name': 'tag-key', 'Values': ['coturn']}])
response = ec2Client.describe_instances(Filters=[{'Name': 'tag-key', 'Values': ['tug-of-war']}])
print(response) print(response)
reservations = response['Reservations'] reservations = response['Reservations']
for reservation in reservations: for reservation in reservations:
@ -28,18 +28,6 @@ print("Delete old security group...")
print("------------------------------------") print("------------------------------------")
try: try:
response = ec2Client.delete_security_group(GroupName='coturn')
response = ec2Client.delete_security_group(GroupName='tug-of-war')
except ClientError as e: except ClientError as e:
print(e) print(e)
print("Delete old elastic ips...")
print("------------------------------------")
try:
response = ec2Client.describe_addresses(Filters=[{'Name': 'tag-key', 'Values': ['coturn']}])
addresses = response['Addresses']
for address in addresses:
ec2Client.release_address(AllocationId=address['AllocationId'])
except ClientError as e:
print(e)
Write Preview
Loading…
Cancel
Save