From ceb16d185f2c1188c9ce75d66722f3a1c626f814 Mon Sep 17 00:00:00 2001 From: Raghav2211 Date: Fri, 16 Jun 2023 16:27:08 +0100 Subject: [PATCH 1/2] initial commit --- provider/aws/aws.go | 3 + provider/aws/cli/services/ec2.go | 25 +++++-- provider/aws/services/ec2/executor.go | 15 ++++- provider/aws/services/ec2/fetcher.go | 93 ++++++++++++++++++++++----- provider/aws/services/ec2/viewer.go | 5 ++ 5 files changed, 116 insertions(+), 25 deletions(-) diff --git a/provider/aws/aws.go b/provider/aws/aws.go index 2860a02..744d237 100644 --- a/provider/aws/aws.go +++ b/provider/aws/aws.go @@ -7,6 +7,7 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -29,6 +30,7 @@ var ( type Client struct { EC2 *ec2.EC2 S3 *s3.S3 + Cloudwatch *cloudwatch.CloudWatch S3Downloader *s3manager.Downloader } @@ -37,6 +39,7 @@ func NewClient(profile, region string, debug bool) (client *Client) { client = &Client{ EC2: ec2.New(session), S3: s3.New(session), + Cloudwatch: cloudwatch.New(session), S3Downloader: s3manager.NewDownloader(session), } return diff --git a/provider/aws/cli/services/ec2.go b/provider/aws/cli/services/ec2.go index 58cb43f..6d4d9c5 100644 --- a/provider/aws/cli/services/ec2.go +++ b/provider/aws/cli/services/ec2.go @@ -3,7 +3,6 @@ package services import ( "cloudctl/provider/aws/cli/globals" "cloudctl/provider/aws/services/ec2" - "log" ) type eC2ListCmd struct { @@ -20,12 +19,16 @@ type instanceDefinitionCmd struct { Id string `name:"name" arg:"required"` } +type ec2DescribeStatisticsCmd struct { +} + type EC2Command struct { - List eC2ListCmd `name:"ls" cmd:"" help:"List ec2 instances"` - InstacneDefinition instanceDefinitionCmd `name:"def" cmd:"" help:"Get ec2 instance definition"` + List eC2ListCmd `name:"ls" cmd:"" help:"List ec2 instances"` + InstacneDefinition instanceDefinitionCmd `name:"def" cmd:"" help:"Get ec2 instance definition"` + DescribeStatistics ec2DescribeStatisticsCmd `name:"stats" cmd:"" help:"Get ec2 instance(s) statistics"` } -func (cmd *eC2ListCmd) Run(globals *globals.CLIFlag) error { +func (cmd eC2ListCmd) Run(globals *globals.CLIFlag) error { filters := []ec2.InstanceListFilterOptFunc{ ec2.WithAvailabilityZone(cmd.AvailabilityZones), @@ -51,9 +54,17 @@ func (cmd *eC2ListCmd) Run(globals *globals.CLIFlag) error { } -func (cmd *instanceDefinitionCmd) Run(globals *globals.CLIFlag) error { - log.Default().Println("get definition for :", cmd.Id) - icmd := ec2.NewinstanceDescribeCommandExecutor(globals, cmd.Id) +func (cmd instanceDefinitionCmd) Run(globals *globals.CLIFlag) error { + icmd := ec2.NewInstanceDescribeCommandExecutor(globals, cmd.Id) + err := icmd.Execute() + if err != nil { + return err + } + return nil +} + +func (cmd ec2DescribeStatisticsCmd) Run(globals *globals.CLIFlag) error { + icmd := ec2.NewEC2StatisticsDescribeCommandExecutor(globals) err := icmd.Execute() if err != nil { return err diff --git a/provider/aws/services/ec2/executor.go b/provider/aws/services/ec2/executor.go index a60aed6..bd99764 100644 --- a/provider/aws/services/ec2/executor.go +++ b/provider/aws/services/ec2/executor.go @@ -5,6 +5,7 @@ import ( "cloudctl/provider/aws" "cloudctl/provider/aws/cli/globals" "cloudctl/time" + "fmt" "strings" ) @@ -20,7 +21,7 @@ func NewinstanceListCommandExecutor(flag *globals.CLIFlag, filter InstanceListFi } } -func NewinstanceDescribeCommandExecutor(flag *globals.CLIFlag, instanceId string) *executor.CommandExecutor { +func NewInstanceDescribeCommandExecutor(flag *globals.CLIFlag, instanceId string) *executor.CommandExecutor { client := aws.NewClient(flag.Profile, flag.Region, flag.Debug) spaceTrimmedInstanceId := strings.TrimSpace(instanceId) return &executor.CommandExecutor{ @@ -32,3 +33,15 @@ func NewinstanceDescribeCommandExecutor(flag *globals.CLIFlag, instanceId string Viewer: instanceInfoViewer, } } + +func NewEC2StatisticsDescribeCommandExecutor(flag *globals.CLIFlag) *executor.CommandExecutor { + fmt.Println("NewEC2StatisticsDescribeCommandExecutor") + client := aws.NewClient(flag.Profile, flag.Region, flag.Debug) + return &executor.CommandExecutor{ + Fetcher: &statisticsFetcher{ + client: client, + tz: time.GetTZ(flag.TZShortIdentifier), + }, + Viewer: ec2StatisticsViewer, + } +} diff --git a/provider/aws/services/ec2/fetcher.go b/provider/aws/services/ec2/fetcher.go index 43fcbe3..124145b 100644 --- a/provider/aws/services/ec2/fetcher.go +++ b/provider/aws/services/ec2/fetcher.go @@ -1,42 +1,50 @@ package ec2 import ( - "cloudctl/provider/aws" - "cloudctl/time" + ctlaws "cloudctl/provider/aws" + ctltime "cloudctl/time" "cloudctl/viewer" "errors" "fmt" "log" "sync" + "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/ec2" ) type instanceListFetcher struct { - client *aws.Client - tz *time.Timezone + client *ctlaws.Client + tz *ctltime.Timezone filter InstanceListFilter } type instanceDefinitionFetcher struct { - client *aws.Client - tz *time.Timezone + client *ctlaws.Client + tz *ctltime.Timezone id *string } +type statisticsFetcher struct { + client *ctlaws.Client + tz *ctltime.Timezone +} + func (f instanceListFetcher) Fetch() interface{} { apiOutput, err := fetchInstanceList(f.client, f.filter) instancesByState := make(map[string][]*instanceSummary) if len(*apiOutput) == 0 { - errorInfo := aws.NewErrorInfo(NoInstanceFound(), viewer.INFO, nil) + errorInfo := ctlaws.NewErrorInfo(NoInstanceFound(), viewer.INFO, nil) return &instanceListOutput{instancesByState: instancesByState, err: errorInfo} } for _, o := range *apiOutput { instancesByState[*o.State.Name] = append(instancesByState[*o.State.Name], newInstanceSummary(o, f.tz)) } if err != nil { - errorInfo := aws.NewErrorInfo(aws.AWSError(err), viewer.ERROR, nil) + errorInfo := ctlaws.NewErrorInfo(ctlaws.AWSError(err), viewer.ERROR, nil) return &instanceListOutput{instancesByState: instancesByState, err: errorInfo} } return &instanceListOutput{instancesByState: instancesByState, err: nil} @@ -50,10 +58,61 @@ func (f instanceDefinitionFetcher) Fetch() interface{} { return definition } -func fetchInstanceList(client *aws.Client, instanceListFilter InstanceListFilter) (*[]*ec2.Instance, error) { - var fetch func(filter []*ec2.Filter, nextMarker string, instances *[]*ec2.Instance, client *aws.Client) error +func (f statisticsFetcher) Fetch() interface{} { + fmt.Println("in statisticsFetcher") + instanceListFetcher := &instanceListFetcher{ + client: f.client, + tz: f.tz, + filter: *NewInstanceFilter(WithInstanceStates([]string{"running"})), + } + + output := instanceListFetcher.Fetch().(*instanceListOutput) + runningInstances := output.instancesByState["running"] + runningInstancesLen := len(runningInstances) + if runningInstancesLen == 0 { + return NoInstanceFound() + } + + wg := new(sync.WaitGroup) + wg.Add(runningInstancesLen) + + resuts := []*cloudwatch.GetMetricStatisticsOutput{} + currentTime := time.Now() + startTime := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day()-15, 00, 00, 00, 00, currentTime.Location()) + startTimeInUnix := startTime.Unix() + + fmt.Println("startTimeInUnix ", startTimeInUnix) + + for _, instance := range runningInstances { + statsInput := cloudwatch.GetMetricStatisticsInput{ + Dimensions: []*cloudwatch.Dimension{ + { + Name: aws.String("InstanceId"), + Value: aws.String(*instance.id), + }, + }, + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Period: aws.Int64(300), + Statistics: []*string{aws.String("Average"), aws.String("Maximum"), aws.String("Minimum")}, + StartTime: &startTime, + EndTime: ¤tTime, + } + go func() { + defer wg.Done() + fmt.Println("fetching for ", *instance.id) + result, _ := f.client.Cloudwatch.GetMetricStatistics(&statsInput) + resuts = append(resuts, result) + }() + } + wg.Wait() + return resuts +} + +func fetchInstanceList(client *ctlaws.Client, instanceListFilter InstanceListFilter) (*[]*ec2.Instance, error) { + var fetch func(filter []*ec2.Filter, nextMarker string, instances *[]*ec2.Instance, client *ctlaws.Client) error - fetch = func(filter []*ec2.Filter, nextMarker string, instances *[]*ec2.Instance, client *aws.Client) error { + fetch = func(filter []*ec2.Filter, nextMarker string, instances *[]*ec2.Instance, client *ctlaws.Client) error { result, err := client.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ Filters: filter, NextToken: &nextMarker, @@ -86,7 +145,7 @@ func fetchInstanceList(client *aws.Client, instanceListFilter InstanceListFilter return &instances, err } -func fetchInstanceDefinition(instanceId *string, tz *time.Timezone, client *aws.Client) (*instanceDefinition, error) { +func fetchInstanceDefinition(instanceId *string, tz *ctltime.Timezone, client *ctlaws.Client) (*instanceDefinition, error) { instanceDefinition := newInstanceDefinition() networkinterfaces := []*instanceNetworkinterface{} wg := new(sync.WaitGroup) @@ -128,7 +187,7 @@ func fetchInstanceDefinition(instanceId *string, tz *time.Timezone, client *aws. wg.Wait() return instanceDefinition, nil } -func fetchInstacneDetail(instanceId *string, client *aws.Client) chan []*ec2.Reservation { +func fetchInstacneDetail(instanceId *string, client *ctlaws.Client) chan []*ec2.Reservation { instancesChan := make(chan []*ec2.Reservation) go func() { defer close(instancesChan) @@ -142,7 +201,7 @@ func fetchInstacneDetail(instanceId *string, client *aws.Client) chan []*ec2.Res return instancesChan } -func fetchInstanceVolumeSummary(volumemappings []*ec2.InstanceBlockDeviceMapping, client *aws.Client) *instanceVolumeSummary { +func fetchInstanceVolumeSummary(volumemappings []*ec2.InstanceBlockDeviceMapping, client *ctlaws.Client) *instanceVolumeSummary { volumeIds := []*string{} volumes := []*instanceVolume{} for _, b := range volumemappings { @@ -151,7 +210,7 @@ func fetchInstanceVolumeSummary(volumemappings []*ec2.InstanceBlockDeviceMapping data, err := client.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: volumeIds}) if err != nil { log.Fatalf("error occurred in fetchInstanceVolumeSummary %s", err) - errorInfo := aws.NewErrorInfo(aws.AWSError(err), viewer.ERROR, nil) + errorInfo := ctlaws.NewErrorInfo(ctlaws.AWSError(err), viewer.ERROR, nil) return newInstanceVolumeSummary(volumes, errorInfo) } for _, volume := range data.Volumes { @@ -160,7 +219,7 @@ func fetchInstanceVolumeSummary(volumemappings []*ec2.InstanceBlockDeviceMapping return newInstanceVolumeSummary(volumes, nil) } -func fetchIngressEgressRuleSummary(enis []*ec2.InstanceNetworkInterface, client *aws.Client) *instanceIngressEgressRuleSummary { +func fetchIngressEgressRuleSummary(enis []*ec2.InstanceNetworkInterface, client *ctlaws.Client) *instanceIngressEgressRuleSummary { securityGroupIds := []*string{} for _, eni := range enis { for _, sg := range eni.Groups { @@ -170,7 +229,7 @@ func fetchIngressEgressRuleSummary(enis []*ec2.InstanceNetworkInterface, client data, err := client.EC2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{GroupIds: securityGroupIds}) if err != nil { log.Fatalf("error occured fetchIngressEgressRuleSummary %s", err) - errorInfo := aws.NewErrorInfo(aws.AWSError(err), viewer.ERROR, nil) + errorInfo := ctlaws.NewErrorInfo(ctlaws.AWSError(err), viewer.ERROR, nil) return &instanceIngressEgressRuleSummary{apiError: errorInfo} } diff --git a/provider/aws/services/ec2/viewer.go b/provider/aws/services/ec2/viewer.go index 45285bd..762564c 100644 --- a/provider/aws/services/ec2/viewer.go +++ b/provider/aws/services/ec2/viewer.go @@ -125,6 +125,11 @@ func instanceInfoViewer(o interface{}) viewer.Viewer { return cTviewer } +func ec2StatisticsViewer(o interface{}) viewer.Viewer { + // fmt.Println("data ", o) + return &viewer.CompoundViewer{} +} + func renderInstanceSummary(o *instanceSummary) *viewer.TableViewer { tViewer := viewer.NewTableViewer() From 3522b63d320b6e858420818b2820fdf38389c126 Mon Sep 17 00:00:00 2001 From: Raghav <7526431+Raghav2211@users.noreply.github.com> Date: Fri, 22 Sep 2023 14:51:15 +0100 Subject: [PATCH 2/2] wip --- provider/aws/services/ec2/executor.go | 2 - provider/aws/services/ec2/fetcher.go | 113 ++++++++++++++++++++------ provider/aws/services/ec2/model.go | 25 +++++- provider/aws/services/ec2/viewer.go | 25 +++++- 4 files changed, 133 insertions(+), 32 deletions(-) diff --git a/provider/aws/services/ec2/executor.go b/provider/aws/services/ec2/executor.go index bd99764..9eabca0 100644 --- a/provider/aws/services/ec2/executor.go +++ b/provider/aws/services/ec2/executor.go @@ -5,7 +5,6 @@ import ( "cloudctl/provider/aws" "cloudctl/provider/aws/cli/globals" "cloudctl/time" - "fmt" "strings" ) @@ -35,7 +34,6 @@ func NewInstanceDescribeCommandExecutor(flag *globals.CLIFlag, instanceId string } func NewEC2StatisticsDescribeCommandExecutor(flag *globals.CLIFlag) *executor.CommandExecutor { - fmt.Println("NewEC2StatisticsDescribeCommandExecutor") client := aws.NewClient(flag.Profile, flag.Region, flag.Debug) return &executor.CommandExecutor{ Fetcher: &statisticsFetcher{ diff --git a/provider/aws/services/ec2/fetcher.go b/provider/aws/services/ec2/fetcher.go index 124145b..564959e 100644 --- a/provider/aws/services/ec2/fetcher.go +++ b/provider/aws/services/ec2/fetcher.go @@ -59,7 +59,6 @@ func (f instanceDefinitionFetcher) Fetch() interface{} { } func (f statisticsFetcher) Fetch() interface{} { - fmt.Println("in statisticsFetcher") instanceListFetcher := &instanceListFetcher{ client: f.client, tz: f.tz, @@ -67,46 +66,35 @@ func (f statisticsFetcher) Fetch() interface{} { } output := instanceListFetcher.Fetch().(*instanceListOutput) + if output.err != nil { + return &instanceStatisticsListOutput{apiError: ctlaws.NewErrorInfo(output.err.Err, viewer.INFO, nil)} + } runningInstances := output.instancesByState["running"] runningInstancesLen := len(runningInstances) if runningInstancesLen == 0 { - return NoInstanceFound() + return &instanceStatisticsListOutput{apiError: ctlaws.NewErrorInfo(NoInstanceFound(), viewer.INFO, nil)} } wg := new(sync.WaitGroup) wg.Add(runningInstancesLen) - resuts := []*cloudwatch.GetMetricStatisticsOutput{} + instancesStatsOutput := []instanceStatisticsOutput{} + currentTime := time.Now() - startTime := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day()-15, 00, 00, 00, 00, currentTime.Location()) - startTimeInUnix := startTime.Unix() - - fmt.Println("startTimeInUnix ", startTimeInUnix) - - for _, instance := range runningInstances { - statsInput := cloudwatch.GetMetricStatisticsInput{ - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("InstanceId"), - Value: aws.String(*instance.id), - }, - }, - MetricName: aws.String("CPUUtilization"), - Namespace: aws.String("AWS/EC2"), - Period: aws.Int64(300), - Statistics: []*string{aws.String("Average"), aws.String("Maximum"), aws.String("Minimum")}, - StartTime: &startTime, - EndTime: ¤tTime, - } + startTime := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day()-2, 00, 00, 00, 00, currentTime.Location()) + + for _, v := range runningInstances { + instance := v go func() { defer wg.Done() - fmt.Println("fetching for ", *instance.id) - result, _ := f.client.Cloudwatch.GetMetricStatistics(&statsInput) - resuts = append(resuts, result) + stats := fetchInstanceStatistics(*instance.id, startTime, currentTime, f.client) + instancesStatsOutput = append(instancesStatsOutput, *stats) }() } wg.Wait() - return resuts + return &instanceStatisticsListOutput{ + stats: instancesStatsOutput, + } } func fetchInstanceList(client *ctlaws.Client, instanceListFilter InstanceListFilter) (*[]*ec2.Instance, error) { @@ -242,3 +230,74 @@ func fetchIngressEgressRuleSummary(enis []*ec2.InstanceNetworkInterface, client return &instanceIngressEgressRuleSummary{ingressRules: ingressRules, egressRules: egressRules} } + +func fetchInstanceStatistics(instanceId string, startTime, currentTime time.Time, client *ctlaws.Client) *instanceStatisticsOutput { + statsInput := cloudwatch.GetMetricStatisticsInput{ + Dimensions: []*cloudwatch.Dimension{ + { + Name: aws.String("InstanceId"), + Value: aws.String(instanceId), + }, + }, + MetricName: aws.String("CPUUtilization"), + Namespace: aws.String("AWS/EC2"), + Period: aws.Int64(300), + Statistics: []*string{aws.String("Average"), aws.String("Maximum"), aws.String("Minimum")}, + StartTime: &startTime, + EndTime: ¤tTime, + } + result, err := client.Cloudwatch.GetMetricStatistics(&statsInput) + if err != nil { + return &instanceStatisticsOutput{apiError: ctlaws.NewErrorInfo(ctlaws.AWSError(err), viewer.ERROR, nil)} + } + // fmt.Println("result ", result) + + var min float64 = 1.7e+308 + var max float64 = 0.0 + var total float64 = 0.0 + + freq := map[string]int{} + + for _, metricStats := range result.Datapoints { + + if instanceId == "i-0f20c6907aa0e6b6f" { + fmt.Println("metricStats == ", metricStats) + } + // fmt.Println("Minimum == ", *metricStats.Minimum) + if *metricStats.Minimum < min { + min = *metricStats.Minimum + } + if *metricStats.Maximum > max { + max = *metricStats.Maximum + } + total = total + *metricStats.Average + + if *metricStats.Minimum <= 45 || *metricStats.Average <= 45 || *metricStats.Maximum <= 45 { + freq[string(CPU_LOW)] = freq[string(CPU_LOW)] + 1 + } + if (*metricStats.Minimum > 45 && *metricStats.Minimum <= 75) || (*metricStats.Average > 45 && *metricStats.Average <= 75) || (*metricStats.Maximum > 45 && *metricStats.Maximum <= 75) { + freq[string(CPU_MODERATE)] = freq[string(CPU_MODERATE)] + 1 + } + if *metricStats.Minimum > 75 || *metricStats.Average > 75 || *metricStats.Maximum > 75 { + freq[string(CPU_HIGH)] = freq[string(CPU_HIGH)] + 1 + } + } + avg := total / float64(len(result.Datapoints)) + + status := CPU_LOW + if avg > 45 && avg <= 75 { + status = CPU_MODERATE + } + if avg > 75 { + status = CPU_HIGH + } + + fmt.Println("freq for ", instanceId, "=", freq, " datapointlen= ", len(result.Datapoints)) + return &instanceStatisticsOutput{ + instanceId: &instanceId, + Average: &avg, + Maximum: &max, + Minimum: &min, + CPUStatus: status, + } +} diff --git a/provider/aws/services/ec2/model.go b/provider/aws/services/ec2/model.go index d10ee9f..e9809ae 100644 --- a/provider/aws/services/ec2/model.go +++ b/provider/aws/services/ec2/model.go @@ -10,10 +10,33 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" ) +type CPUUtilizationStatus string + const ( - NO_VALUE string = "-" + NO_VALUE string = "-" + CPU_LOW CPUUtilizationStatus = "Low" + CPU_MODERATE CPUUtilizationStatus = "Moderate" + CPU_HIGH CPUUtilizationStatus = "High" ) +type instanceStatisticsOutput struct { + instanceId *string + + Average *float64 + Maximum *float64 + Minimum *float64 + + CPUStatus CPUUtilizationStatus + // Timestamp *time.Time // time between start to end + + apiError *aws.ErrorInfo +} + +type instanceStatisticsListOutput struct { + stats []instanceStatisticsOutput + apiError *aws.ErrorInfo +} + type ingressRule struct { portRange *string protocol *string diff --git a/provider/aws/services/ec2/viewer.go b/provider/aws/services/ec2/viewer.go index 762564c..f3c4d65 100644 --- a/provider/aws/services/ec2/viewer.go +++ b/provider/aws/services/ec2/viewer.go @@ -77,6 +77,13 @@ var ( "deleteOnTermination", "securityGroups", } + instanceStatisticsTableHeader = viewer.Row{ + "instanceId", + "Minimum", + "Average", + "Maximum", + "Status", + } ) func instanceListViewer(o interface{}) viewer.Viewer { @@ -126,8 +133,22 @@ func instanceInfoViewer(o interface{}) viewer.Viewer { } func ec2StatisticsViewer(o interface{}) viewer.Viewer { - // fmt.Println("data ", o) - return &viewer.CompoundViewer{} + data := o.(*instanceStatisticsListOutput) + tViewer := viewer.NewTableViewer() + tViewer.AddHeader(instanceStatisticsTableHeader) + tViewer.SetTitle("Statistics") + + for _, stats := range data.stats { + tViewer.AddRow(viewer.Row{ + *stats.instanceId, + *stats.Minimum, + *stats.Average, + *stats.Maximum, + stats.CPUStatus, + }) + } + // return tViewer + return viewer.NewTableViewer() } func renderInstanceSummary(o *instanceSummary) *viewer.TableViewer {