I was curious how others deal and monitor TCP Port Exhaustion with either their primary or additional polling engine servers. We occassionally have servers where a ridiculous amount of TCP connections are established, but are never released. In most circumstances, we unmanage the node, reboot the APEs, and then reboot the offending server when we can. After the offendeing server has been rebooted, we end up re-managing it. We rarely see the same server as a repeat offender. This is just something we see pop up every couple of months, if that.
I ended up writing a Powershell based SAM monitor to get back all of the various connection states to try and see what is going on, as well as tell me which servers are exceeding 200 connections. If anyone is interested or have recommendations on improving the query, please let me know.
#Retrieve TCP Connections
Try {
$TCPConnections = Get-NetTCPConnection -ErrorAction Stop | Select-Object LocalAddress,LocalPort,RemoteAddress,RemotePort,CreationTime,State,OwningProcess
} Catch {
Exit 1
}
#Group Connections
$HostSummary = $TCPConnections | Group-Object -Property RemoteAddress | Select-Object Name,Count | Sort-Object -Property Count -Descending
$HighUtilization = $HostSummary | Where-Object {$_.Count -gt 200}
$UtilizationArray = [System.Collections.ArrayList]@()
ForEach ($HighUtilizationHost in $HighUtilization) {
$UtilizationArray.Add(("IP: $($HighUtilizationHost.Name),Count: $($HighUtilizationHost.Count)")) | Out-Null
}
#Output
#Total
Write-Host "Statistic.Total: $(($TCPConnections | Measure-Object).Count)"
Write-Host "Message.Total: Currently $(($TCPConnections | Measure-Object).Count) connections open"
#High Utilzation
Write-host "Statistic.HighUtilizationHosts: $(($HighUtilization | Measure-Object).Count)"
If ([String]::IsNullOrEmpty($UtilizationArray)) {
Write-Host "Message.HighUtilizationHosts: No systems exceeding 200 connections"
} Else {
Write-Host "Message.HighUtilizationHosts: The following systems are showing high connenections $($UtilizationArray -join "-")"
}
#States
Write-Host "Statistic.ConnectionEstablished: $(($TCPConnections | Where-Object {$_.State -eq "Established"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionTimeWait: $(($TCPConnections | Where-Object {$_.State -eq "TimeWait"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionSynSent: $(($TCPConnections | Where-Object {$_.State -eq "SynSent"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionBound: $(($TCPConnections | Where-Object {$_.State -eq "Bound"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionClosed: $(($TCPConnections | Where-Object {$_.State -eq "Closed"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionCloseWait: $(($TCPConnections | Where-Object {$_.State -eq "CloseWait"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionClosing: $(($TCPConnections | Where-Object {$_.State -eq "Closing"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionDeleteTCB: $(($TCPConnections | Where-Object {$_.State -eq "DeleteTCB"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionFinWait: $(($TCPConnections | Where-Object {$_.State -like "FinWait*"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionLastAck: $(($TCPConnections | Where-Object {$_.State -eq "LastAck"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionListen: $(($TCPConnections | Where-Object {$_.State -eq "Listen"} | Measure-Object).count)"
Write-Host "Statistic.ConnectionSynReceived: $(($TCPConnections | Where-Object {$_.State -eq "SynReceived"} | Measure-Object).count)"
Exit 0
