wip

api/Readme.md

@@ -1,6 +1,3 @@
-
-
-
 `sudo bash ./install.sh`
 
 to redo all the steps remove the lock files
@@ -19,3 +16,80 @@ export ROOT=/mnt/data1/swarms;
 sudo rm ${ROOT}/opt/swarms/install/*; 
 sudo bash ./install.sh
 ```
+* setup
+To install on linux: 
+https://docs.aws.amazon.com/systems-manager/
+
+```
+curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb"
+sudo dpkg  -i ./session-manager-plugin.deb 
+```
+
+* run
+
+To redo the installation steps for the Swarms tool on your system, follow these commands sequentially:
+
+1. Set the ROOT variable:
+   ```bash
+   export ROOT=/mnt/data1/swarms
+   ```
+
+2. Remove the lock files:
+   ```bash
+   sudo rm ${ROOT}/opt/swarms/install/*
+   ```
+
+3. Run the installation script again:
+   ```bash
+   sudo bash ./install.sh
+   ```
+
+For setting up the Session Manager plugin on Linux, you can follow these commands:
+
+1. Download the Session Manager plugin:
+   ```bash
+   curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb"
+   ```
+
+2. Install the plugin:
+   ```bash
+   sudo dpkg -i ./session-manager-plugin.deb
+   ```
+
+After that, you can run your desired commands or workflows.** get the instance id
+`aws ec2 describe-instances`
+
+** start a session
+`aws ssm start-session --target  i-XXXX`
+
+** on the machine:
+```
+sudo su - 
+tail /var/log/cloud-init-output.log 
+```
+
+Convert this to an automation of your choice to run all the steps
+and run this on all the instances
+
+To get the instance ID and start a session using AWS CLI, follow these steps:
+
+1. **Get the Instance ID:**
+   Run the following command to list your instances and their details:
+   ```bash
+   aws ec2 describe-instances
+   ```
+
+2. **Start a Session:**
+   Replace `i-XXXX` with your actual instance ID from the previous step:
+   ```bash
+   aws ssm start-session --target i-XXXX
+   ```
+
+3. **On the Machine:**
+   After starting the session, you can execute the following commands:
+   ```bash
+   sudo su -
+   tail /var/log/cloud-init-output.log
+   ```
+
+

api/check_status.py

@@ -0,0 +1,52 @@
+import os
+import json
+import boto3
+
+# Create .cache directory if it doesn't exist
+os.makedirs('.cache', exist_ok=True)
+
+def cache(name, value):
+    cache_file = f'.cache/{name}'
+    if not os.path.isfile(cache_file):
+        with open(cache_file, 'w') as f:
+            f.write(value)
+
+# Initialize Boto3 SSM client
+ssm = boto3.client('ssm')
+
+# List commands from AWS SSM
+response = ssm.list_commands()
+
+cache("aws_ssm_list_commands", response)
+
+# Retrieve commands
+print(response)
+commands = response["Commands"]
+run_ids = [cmd['CommandId'] for cmd in commands]
+print(f"RUNIDS: {run_ids}")
+
+# Check the status of each command
+for command in commands:
+    #print(command)
+    command_id = command['CommandId']
+    status = command['Status']
+    #eG: command= {'CommandId': '820dcf47-e8d7-4c23-8e8a-bc64de2883ff', 'DocumentName': 'AWS-RunShellScript', 'DocumentVersion': '$DEFAULT', 'Comment': '', 'ExpiresAfter': datetime.datetime(2024, 12, 13, 12, 41, 24, 683000, tzinfo=tzlocal()), 'Parameters': {'commands': ['sudo su - -c "tail /var/log/cloud-init-output.log"']}, 'InstanceIds': [], 'Targets': [{'Key': 'instanceids', 'Values': ['i-073378237c5a9dda1']}], 'RequestedDateTime': datetime.datetime(2024, 12, 13, 10, 41, 24, 683000, tzinfo=tzlocal()), 'Status': 'Success', 'StatusDetails': 'Success', 'OutputS3Region': 'us-east-1', 'OutputS3BucketName': '', 'OutputS3KeyPrefix': '', 'MaxConcurrency': '50', 'MaxErrors': '0', 'TargetCount': 1, 'CompletedCount': 1, 'ErrorCount': 0, 'DeliveryTimedOutCount': 0, 'ServiceRole': '', 'NotificationConfig': {'NotificationArn': '', 'NotificationEvents': [], 'NotificationType': ''}, 'CloudWatchOutputConfig': {'CloudWatchLogGroupName': '', 'CloudWatchOutputEnabled': False}, 'TimeoutSeconds': 3600, 'AlarmConfiguration': {'IgnorePollAlarmFailure': False, 'Alarms': []}, 'TriggeredAlarms': []}], 'ResponseMetadata': {'RequestId': '535839c4-9b87-4526-9c01-ed57f07d21ef', 'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'Server', 'date': 'Fri, 13 Dec 2024 16:58:53 GMT', 'content-type': 'application/x-amz-json-1.1', 'content-length': '2068', 'connection': 'keep-alive', 'x-amzn-requestid': '535839c4-9b87-4526-9c01-ed57f07d21ef'}, 'RetryAttempts': 0}}
+    
+    if status == "Success":
+        print(f"Check logs of {command_id}")
+        # use ssm to  fetch logs using CommandId
+
+        # Assuming you have the command_id from the previous command output
+        command_id = command['CommandId']
+        instance_id = command['Targets'][0]['Values'][0]  # Get the instance ID
+
+        # Fetching logs using CommandId
+        log_response = ssm.get_command_invocation(
+            CommandId=command_id,
+            InstanceId=instance_id
+        )
+        print(log_response['StandardOutputContent'])  # Output logs
+        print(log_response['StandardErrorContent'])    # Error logs (if any)
+        print(f"aws ssm start-session --target  {instance_id}")
+
+

api/fetch_logs.py

@@ -0,0 +1,17 @@
+import json
+
+# Read JSON data from the file
+with open('.cache/aws_ssm_list_commands', 'r') as file:
+    data = json.load(file)
+
+# Loop through each command
+for command in data.get("Commands", []):
+    command_id = command.get("CommandId")
+    status = command.get("Status")
+
+    # Check if the status is 'Success'
+    if status == "Success":
+        # Fetch logs of the command (you may need to implement this function)
+        print(f"Fetching logs for Command ID: {command_id}")
+        # Example: fetch_logs(command_id)
+        # use aws ssm to fetch the logs

api/get_logs.sh

@@ -0,0 +1,43 @@
+rewrite in python boto3
+mkdir -p .cache
+
+cache(){
+    name=".cache/${1}"
+    value="${2}"
+    if [ ! -f $name ];
+    then
+	echo $value > $name
+    fi
+}
+
+aws_ssm_list_commands=$(aws ssm list-commands)
+cache "aws_ssm_list_commands" "$aws_ssm_list_commands"
+#command_status=$(echo "${aws_ssm_list_commands}" | jq  '.Commands[]|[.CommandId,.Status]')
+commands=$(echo "${aws_ssm_list_commands}" | jq  '.Commands[]')
+#run_ids=$(echo $command_status | jq  .[0] -r)
+run_ids=$(echo $command_status | jq -r)
+echo RUNIDS:$run_ids;
+
+while read -r instance_id state; do
+    if [[ $state == "Success" ]]; then
+	echo "Check $instance_id"
+	#             "DocumentName": "AWS-RunShellScript",
+#             "Parameters": {
+#                 "commands": [
+#                     "sudo su - -c \"tail /var/log/cloud-init-output.log\""
+#             "Status": "Success",
+#             "CompletedCount": 1,
+#             "ErrorCount": 0,
+#             "DeliveryTimedOutCount": 0,
+
+    fi
+done  <<< "$run_ids"
+
+
+
+It seems you might be looking for a complete Python script that replicates the functionality of your provided shell script using Boto3. Here’s an example:
+
+```python
+```
+
+This Python script uses Boto3 to list SSM commands, caches them in a file, and checks the status of each command. Adjust the logic as needed to fit your specific processing requirements.

api/read.sh

@@ -0,0 +1,22 @@
+# #read this file one command at a time and loop over it in python
+# # read out the attributes of each object into variables in the loop
+# # tail .cache/aws_ssm_list_commands
+# # {"CommandId":"820dcf47-e8d7-4c23-8e8a-bc64de2883ff","DocumentName":"AWS-RunShellScript","DocumentVersion":"$DEFAULT","Comment":"","ExpiresAfter":1734111684.683,"Parameters":{"commands":["sudo su - -c \"tail /var/log/cloud-init-output.log\""]},"InstanceIds":[],"Targets":[{"Key":"instanceids","Values":["i-073378237c5a9dda1"]}],"RequestedDateTime":1734104484.683,"Status":"Success","StatusDetails":"Success","OutputS3Region":"us-east-1","OutputS3BucketName":"","OutputS3KeyPrefix":"","MaxConcurrency":"50","MaxErrors":"0","TargetCount":1,"CompletedCount":1,"ErrorCount":0,"DeliveryTimedOutCount":0,"ServiceRole":"","NotificationConfig":{"NotificationArn":"","NotificationEvents":[],"NotificationType":""},"CloudWatchOutputConfig":{"CloudWatchLogGroupName":"","CloudWatchOutputEnabled":false},"TimeoutSeconds":3600,"AlarmConfiguration":{"IgnorePollAlarmFailure":false,"Alarms":[]},"TriggeredAlarms":[]}
+
+# # read this result
+# commands=$(jq -e ".Commands[]" -c .cache/aws_ssm_list_commands )
+# for each command {
+#     id=getid
+#     status=get status
+#     if status == successs:
+#        fetch logs of commnd id
+# }
+
+rewrite in python
+
+Here's a Python script that reads the JSON data from the specified file, extracts the attributes of each command, and processes them in a loop:
+
+```python
+```
+
+Make sure to implement the `fetch_logs(command_id)` function according to your requirements.

api/report_status.md

@@ -0,0 +1,63 @@
+Starting session for instance: i-073378237c5a9dda1
++ aws ssm send-command --document-name AWS-RunShellScript --targets Key=instanceids,Values=i-073378237c5a9dda1 --parameters 'commands=["sudo su - -c \"tail /var/log/cloud-init-output.log\""]'
+{
+    "Command": {
+        "CommandId": "820dcf47-e8d7-4c23-8e8a-bc64de2883ff",
+        "DocumentName": "AWS-RunShellScript",
+        "DocumentVersion": "$DEFAULT",
+        "Comment": "",
+        "ExpiresAfter": 1734111684.683,
+        "Parameters": {
+            "commands": [
+                "sudo su - -c \"tail /var/log/cloud-init-output.log\""
+            ]
+        },
+        "InstanceIds": [],
+        "Targets": [
+            {
+                "Key": "instanceids",
+                "Values": [
+                    "i-073378237c5a9dda1"
+                ]
+            }
+        ],
+        "RequestedDateTime": 1734104484.683,
+        "Status": "Pending",
+        "StatusDetails": "Pending",
+        "OutputS3Region": "us-east-1",
+        "OutputS3BucketName": "",
+        "OutputS3KeyPrefix": "",
+        "MaxConcurrency": "50",
+        "MaxErrors": "0",
+        "TargetCount": 0,
+        "CompletedCount": 0,
+        "ErrorCount": 0,
+        "DeliveryTimedOutCount": 0,
+        "ServiceRole": "",
+        "NotificationConfig": {
+            "NotificationArn": "",
+            "NotificationEvents": [],
+            "NotificationType": ""
+        },
+        "CloudWatchOutputConfig": {
+            "CloudWatchLogGroupName": "",
+            "CloudWatchOutputEnabled": false
+        },
+        "TimeoutSeconds": 3600,
+        "AlarmConfiguration": {
+            "IgnorePollAlarmFailure": false,
+            "Alarms": []
+        },
+        "TriggeredAlarms": []
+    }
+}
+read this and check on the results now pls
+
+
+To check the results of the command you executed using AWS SSM, you can use the following command:
+
+```bash
+aws ssm list-command-invocations --command-id 820dcf47-e8d7-4c23-8e8a-bc64de2883ff --details
+```
+
+This command will provide the status and output of the command executed on your instance. Look for the `Status` and `StandardOutputContent` fields in the response to determine the result of your command.

api/run_logs.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# EDIT: we need to make sure the instance is running
+# Get the list of instance IDs
+instance_ids=$(aws ec2 describe-instances --query "Reservations[*].Instances[*].InstanceId" --output text)
+
+# Loop through each instance ID and start a session
+for instance_id in $instance_ids; do
+    echo "Starting session for instance: $instance_id"
+    
+    # Start a session and execute commands (replace with your commands)
+    aws ssm start-session --target "$instance_id" --document-name "AWS-StartInteractiveCommand" --parameters 'commands=["sudo su -","tail /var/log/cloud-init-output.log"]'
+
+done

api/run_logs2.sh

@@ -0,0 +1,35 @@
+# Get the list of instance IDs and their states
+instances=$(aws ec2 describe-instances --query "Reservations[*].Instances[*].[InstanceId,State.Name]" --output text)
+
+# aws ssm send-command --document-name AWS-RunShellScript --targets Key=instanceids,Values=i-073378237c5a9dda1 --parameters 'commands=["sudo su - -c \"tail /var/log/cloud-init-output.log\""]'
+
+parse_command_id(){
+    # send_command_output
+    local send_command_output=$1
+    echo "$send_command_output" | jq -r '.Command.CommandId'
+}
+
+# Loop through each instance ID and state
+while read -r instance_id state; do
+    if [[ $state == "running" ]]; then
+        echo "Starting session for instance: $instance_id"
+        
+        # Start a session and execute commands (replace with your commands)
+        #aws ssm start-session --target "$instance_id" --document-name "AWS-StartInteractiveCommand" --parameters 'commands=["sudo su -","tail /var/log/cloud-init-output.log"]'
+
+	#--target "$instance_id"
+	send_command_output=$(aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=instanceids,Values=$instance_id" --parameters 'commands=["sudo su - -c \"tail /var/log/cloud-init-output.log\""]')
+
+	
+	# now get the command id
+	command_id=$(parse_command_id send_command_output)
+	
+	# now for 4 times, sleep 1 sec,
+	for i in {1..4}; do
+	    sleep 1
+	    command_status=$(aws ssm list-command-invocations --command-id "$command_id" --details)
+	    echo "$command_status"
+	done
+
+    fi
+done <<< "$instances"