PS - Storage Spaces Direct / Storage Replication - Troubleshooting

on
The following is a collection of Storage Spaces Direct, Storage Replica and random hyper converged cluster commands I've been using.

 Please test, test, test and as always use at your own risk.


 #View any outstanding S2D Jobs to ensure S2D is up to date (should be used before host updates or reboots along with draining the nodes from Failover Cluster Manager)
Get-StorageJob

#Check Virtual Disk to ensure they are all attached and not held by Storage Replication
Get-VirtualDisk

#Check Physical Disk
Get-PhysicalDisk

#Collect a Storage health report
Get-StorageSubSystem -name * | Get-StorageHealthReport

#Check if any health jobs are running (such as a rebalance of the pool)
Get-StorageHealthAction

#Connect any detached Virtual Disks
Get-VirtualDisk | Where-Object -Filter { $_.OperationalStatus -eq "Detached" } | Connect-VirtualDisk
#Rebalance the Spaces allocations in a pool to disks with available capacity (used after replacing physical disks)

Get-StoragePool S2D* | Optimize-StoragePool

#Pause cluster nodes
Suspend-ClusterNode "NODE-01" -Cluster "Cluster_01" -Drain
Suspend-ClusterNode "NODE-02" -Cluster "Cluster_01" -Drain
Suspend-ClusterNode "NODE-03" -Cluster "Cluster_01" -Drain

#Pause cluster nodes
Suspend-ClusterNode "NODE-01" -Cluster "Cluster_02" -Drain
Suspend-ClusterNode "NODE-02" -Cluster "Cluster_02" -Drain
Suspend-ClusterNode "NODE-03" -Cluster "Cluster_02" -Drain


#Remove Network Constraint from Cluster_01
Remove-SRNetworkConstraint -SourceComputerName "Cluster_01" -SourceRGName "rg01" -DestinationComputerName "Cluster_02" -DestinationRGName "rg02"
Remove-SRNetworkConstraint -SourceComputerName "Cluster_01" -SourceRGName "rg03" -DestinationComputerName "Cluster_02" -DestinationRGName "rg04"
Remove-SRNetworkConstraint -SourceComputerName "Cluster_01" -SourceRGName "rg05" -DestinationComputerName "Cluster_02" -DestinationRGName "rg06"

#Remove Network Constraint from Cluster_02
Remove-SRNetworkConstraint -SourceComputerName "Cluster_02" -SourceRGName "rg07" -DestinationComputerName "Cluster_01" -DestinationRGName "rg08"
Remove-SRNetworkConstraint -SourceComputerName "Cluster_02" -SourceRGName "rg09" -DestinationComputerName "Cluster_01" -DestinationRGName "rg10"
Remove-SRNetworkConstraint -SourceComputerName "Cluster_02" -SourceRGName "rg11" -DestinationComputerName "Cluster_01" -DestinationRGName "rg12"

#Remove Partnership from Cluster_01
Remove-SRPartnership -SourceComputerName "Cluster_01" -SourceRGName "rg01" -DestinationComputerName "Cluster_02" -DestinationRGName "rg02"
Remove-SRPartnership -SourceComputerName "Cluster_01" -SourceRGName "rg03" -DestinationComputerName "Cluster_02" -DestinationRGName "rg04"
Remove-SRPartnership -SourceComputerName "Cluster_01" -SourceRGName "rg05" -DestinationComputerName "Cluster_02" -DestinationRGName "rg06"

#Remove Partnership from Cluster_02
Remove-SRPartnership -SourceComputerName "Cluster_02" -SourceRGName "rg07" -DestinationComputerName "Cluster_01" -DestinationRGName "rg08"
Remove-SRPartnership -SourceComputerName "Cluster_02" -SourceRGName "rg09" -DestinationComputerName "Cluster_01" -DestinationRGName "rg10"
Remove-SRPartnership -SourceComputerName "Cluster_02" -SourceRGName "rg11" -DestinationComputerName "Cluster_01" -DestinationRGName "rg12"

#Remove SR Groups from Cluster_01
Remove-SRGroup -Name "rg01" -Verbose
Remove-SRGroup -Name "rg03" -Verbose
Remove-SRGroup -Name "rg05" -Verbose

#Remove SR Groups from Cluster_02
Remove-SRGroup -Name "rg07" -Verbose
Remove-SRGroup -Name "rg09" -Verbose
Remove-SRGroup -Name "rg11" -Verbose

#Clear Orphaned SR Logs (you must be attached to the node where the log volumes are presented within the Failover Cluster Manager)
Clear-SRMetadata -AllLogs -AllPartitions -Verbose -whatif

#Create SR Groups
New-SRPartnership -SourceComputerName Cluster_01 -SourceRGName rg01 -SourceVolumeName c:\ClusterStorage\Volume1 -SourceLogVolumeName I: -DestinationComputerName Cluster_02 -DestinationRGName rg02 -DestinationVolumeName c:\ClusterStorage\Volume7 -DestinationLogVolumeName O:
New-SRPartnership -SourceComputerName Cluster_01 -SourceRGName rg03 -SourceVolumeName c:\ClusterStorage\Volume2 -SourceLogVolumeName J: -DestinationComputerName Cluster_02 -DestinationRGName rg04 -DestinationVolumeName c:\ClusterStorage\Volume9 -DestinationLogVolumeName P:
New-SRPartnership -SourceComputerName Cluster_01 -SourceRGName rg05 -SourceVolumeName c:\ClusterStorage\Volume3 -SourceLogVolumeName K: -DestinationComputerName Cluster_02 -DestinationRGName rg06 -DestinationVolumeName c:\ClusterStorage\Volume11 -DestinationLogVolumeName Q:
New-SRPartnership -SourceComputerName Cluster_02 -SourceRGName rg07 -SourceVolumeName c:\ClusterStorage\Volume1 -SourceLogVolumeName I: -DestinationComputerName Cluster_01 -DestinationRGName rg08 -DestinationVolumeName c:\ClusterStorage\Volume7 -DestinationLogVolumeName O:
New-SRPartnership -SourceComputerName Cluster_02 -SourceRGName rg09 -SourceVolumeName c:\ClusterStorage\Volume2 -SourceLogVolumeName J: -DestinationComputerName Cluster_01 -DestinationRGName rg10 -DestinationVolumeName c:\ClusterStorage\Volume9 -DestinationLogVolumeName P:
New-SRPartnership -SourceComputerName Cluster_02 -SourceRGName rg11 -SourceVolumeName c:\ClusterStorage\Volume3 -SourceLogVolumeName K: -DestinationComputerName Cluster_01 -DestinationRGName rg12 -DestinationVolumeName c:\ClusterStorage\Volume11 -DestinationLogVolumeName Q:

#Configure Constraints
Set-SRNetworkConstraint -SourceComputerName Cluster_01 -SourceRGName rg01 -SourceNWInterface "Cluster Network 5" -DestinationComputerName Cluster_02 -DestinationRGName rg02 -DestinationNWInterface "Cluster Network 5"
Set-SRNetworkConstraint -SourceComputerName Cluster_01 -SourceRGName rg03 -SourceNWInterface "Cluster Network 5" -DestinationComputerName Cluster_02 -DestinationRGName rg04 -DestinationNWInterface "Cluster Network 5"
Set-SRNetworkConstraint -SourceComputerName Cluster_01 -SourceRGName rg05 -SourceNWInterface "Cluster Network 5" -DestinationComputerName Cluster_02 -DestinationRGName rg06 -DestinationNWInterface "Cluster Network 5"
Set-SRNetworkConstraint -SourceComputerName Cluster_02 -SourceRGName rg07 -SourceNWInterface "Cluster Network 5" -DestinationComputerName Cluster_01 -DestinationRGName rg08 -DestinationNWInterface "Cluster Network 5"
Set-SRNetworkConstraint -SourceComputerName Cluster_02 -SourceRGName rg09 -SourceNWInterface "Cluster Network 5" -DestinationComputerName Cluster_01 -DestinationRGName rg10 -DestinationNWInterface "Cluster Network 5"
Set-SRNetworkConstraint -SourceComputerName Cluster_02 -SourceRGName rg11 -SourceNWInterface "Cluster Network 5" -DestinationComputerName Cluster_01 -DestinationRGName rg12 -DestinationNWInterface "Cluster Network 5"

#View the Storage Replica events that show creation of the partnership. This event states the number of copied bytes and the time taken
Get-WinEvent -ProviderName Microsoft-Windows-StorageReplica | Where-Object {$_.ID -eq "1215"} | Format-List

#Determine overview of the replication status
Get-WinEvent -ProviderName Microsoft-Windows-StorageReplica -max 20

#Check the number of bytes remaining to copy.
(Get-SRGroup).Replicas | Select-Object numofbytesremaining

#If required manually start a replica synchronisation
Sync-SRGroup -Name "rg01" -Verbose

#Once initial block copy is complete check the outstanding replication traffic per replication group
do{
    $r=(Get-SRGroup -Name "rg01").replicas
    [System.Console]::Write("Number of remaining bytes {0}`n", $r.NumOfBytesRemaining)
    Start-Sleep 10
}until($r.ReplicationStatus -eq 'ContinuouslyReplicating')
Write-Output "Replica Status: "$r.replicationstatus

#Check Storage Replication Partnership
Get-SRPartnership

#Check Storage Replicaiton Access
Get-SRAccess

#Check and SMB Bandwidth limits that have been configured
Get-SmbBandwidthLimit -Category StorageReplication

#Check Network Interface Adapters
Get-SmbClientNetworkInterface

#Check RDMA Network Adapters
Get-NetAdapterRdma

#Check If VMQ are enable/disabled
Get-NetAdapterVmq | Select Name,InterfaceDescription,Enabled



#Increase existing S2D volume If successful you should be able to refresh Clustered Volume within Failover Cluster Manager and the new size should be presenting the new size in the capacity but the volume would be displayed with old size (as per screenshot).

#Run the following to list virtual disks
Get-VirtualDisk

#Run the following to list Storage Tier Friendly Name  –  where SSC01_Volume_01 is correct name for the volume you wish to increase
Get-VirtualDisk 'SSC01_Volume_01' | Get-StorageTier | Select FriendlyName

#Run the following to increase size of Volume Storage Tier where 6TB is the final size
Resize-StorageTier -InputObject (Get-StorageTier -FriendlyName
"SSC01_Volume_01_Capacity") -Size 6TB

#Run following to list storage jobs to ensure S2D is up to date and working

Get-StorageJob

#If successful you should be able to refresh Clustered Volume within Failover Cluster Manager and the new size should be presenting the new size in the capacity but the volume would be displayed with old size.

#To extend the volume perform the following commands in your PowerShell window where SSC01_Volume_01 is the volume that needs increased
# Choose virtual disk
$VirtualDisk = Get-VirtualDisk SSC01_Volume_01

# Get its partition
$Partition = $VirtualDisk | Get-Disk | Get-Partition | Where PartitionNumber -Eq 2

# Resize to its maximum supported size
$Partition | Resize-Partition -Size ($Partition | Get-PartitionSupportedSize).SizeMax 



################################################################################
#Show-PrettyPool.ps1 - (https://blogs.technet.microsoft.com/filecab/2016/11/21/deep-dive-pool-in-spaces-direct/)

#region Show-PrettyPool.ps1
# Written by Cosmos Darwin, PM
# Copyright (C) 2017 Microsoft Corporation
# MIT License
# 08/2017

Function ConvertTo-PrettyCapacity {
    <#
    .SYNOPSIS Convert raw bytes into prettier capacity strings.
    .DESCRIPTION Takes an integer of bytes, converts to the largest unit (kilo-, mega-, giga-, tera-) that will result in at least 1.0, rounds to given precision, and appends standard unit symbol.
    .PARAMETER Bytes The capacity in bytes.
    .PARAMETER UseBaseTwo Switch to toggle use of binary units and prefixes (mebi, gibi) rather than standard (mega, giga).
    .PARAMETER RoundTo The number of decimal places for rounding, after conversion.
    #>

    Param (
        [Parameter(
            Mandatory = $True,
            ValueFromPipeline = $True
            )
        ]
    [Int64]$Bytes,
    [Int64]$RoundTo = 0,
    [Switch]$UseBaseTwo # Base-10 by Default
    )

    If ($Bytes -Gt 0) {
        $BaseTenLabels = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
        $BaseTwoLabels = ("bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB")
        If ($UseBaseTwo) {
            $Base = 1024
            $Labels = $BaseTwoLabels
        }
        Else {
            $Base = 1000
            $Labels = $BaseTenLabels
        }
        $Order = [Math]::Floor( [Math]::Log($Bytes, $Base) )
        $Rounded = [Math]::Round($Bytes/( [Math]::Pow($Base, $Order) ), $RoundTo)
        [String]($Rounded) + $Labels[$Order]
    }
    Else {
        0
    }
    Return
}


Function ConvertTo-PrettyPercentage {
    <#
    .SYNOPSIS Convert (numerator, denominator) into prettier percentage strings.
    .DESCRIPTION Takes two integers, divides the former by the latter, multiplies by 100, rounds to given precision, and appends "%".
    .PARAMETER Numerator Really?
    .PARAMETER Denominator C'mon.
    .PARAMETER RoundTo The number of decimal places for rounding.
    #>

    Param (
        [Parameter(Mandatory = $True)]
            [Int64]$Numerator,
        [Parameter(Mandatory = $True)]
            [Int64]$Denominator,
        [Int64]$RoundTo = 1
    )

    If ($Denominator -Ne 0) { # Cannot Divide by Zero
        $Fraction = $Numerator/$Denominator
        $Percentage = $Fraction * 100
        $Rounded = [Math]::Round($Percentage, $RoundTo)
        [String]($Rounded) + "%"
    }
    Else {
        0
    }
    Return
}

Function Find-LongestCommonPrefix {
    <#
    .SYNOPSIS Find the longest prefix common to all strings in an array.
    .DESCRIPTION Given an array of strings (e.g. "Seattle", "Seahawks", and "Season"), returns the longest starting substring ("Sea") which is common to all the strings in the array. Not case sensitive.
    .PARAMETER Strings The input array of strings.
    #>

    Param (
        [Parameter(
            Mandatory = $True
            )
        ]
        [Array]$Array
    )

    If ($Array.Length -Gt 0) {

        $Exemplar = $Array[0]

        $PrefixEndsAt = $Exemplar.Length # Initialize
        0..$Exemplar.Length | ForEach {
            $Character = $Exemplar[$_]
            ForEach ($String in $Array) {
                If ($String[$_] -Eq $Character) {
                    # Match
                }
                Else {
                    $PrefixEndsAt = [Math]::Min($_, $PrefixEndsAt)
                }
            }
        }
        # Prefix
        $Exemplar.SubString(0, $PrefixEndsAt)
    }
    Else {
        # None
    }
    Return
}

Function Reverse-String {
    <#
    .SYNOPSIS Takes an input string ("Gates") and returns the character-by-character reversal ("setaG").
    #>

    Param (
        [Parameter(
            Mandatory = $True,
            ValueFromPipeline = $True
            )
        ]
        $String
    )

    $Array = $String.ToCharArray()
    [Array]::Reverse($Array)
    -Join($Array)
    Return
}

Function New-UniqueRootLookup {
    <#
    .SYNOPSIS Creates hash table that maps strings, particularly server names of the form [CommonPrefix][Root][CommonSuffix], to their unique Root.
    .DESCRIPTION For example, given ("Server-A2.Contoso.Local", "Server-B4.Contoso.Local", "Server-C6.Contoso.Local"), returns key-value pairs:
        {
        "Server-A2.Contoso.Local" -> "A2"
        "Server-B4.Contoso.Local" -> "B4"
        "Server-C6.Contoso.Local" -> "C6"
        }
    .PARAMETER Strings The keys of the hash table.
    #>

    Param (
        [Parameter(
            Mandatory = $True
            )
        ]
        [Array]$Strings
    )

    # Find Prefix

    $CommonPrefix = Find-LongestCommonPrefix $Strings

    # Find Suffix

    $ReversedArray = @()
    ForEach($String in $Strings) {
        $ReversedString = $String | Reverse-String
        $ReversedArray += $ReversedString
    }

    $CommonSuffix = $(Find-LongestCommonPrefix $ReversedArray) | Reverse-String

    # String -> Root Lookup

    $Lookup = @{}
    ForEach($String in $Strings) {
        $Lookup[$String] = $String.Substring($CommonPrefix.Length, $String.Length - $CommonPrefix.Length - $CommonSuffix.Length)
    }

    $Lookup
    Return
}

### SCRIPT... ###

$Nodes = Get-StorageSubSystem Cluster* | Get-StorageNode
$Drives = Get-StoragePool S2D* | Get-PhysicalDisk

$Names = @()
ForEach ($Node in $Nodes) {
    $Names += $Node.Name
}

$UniqueRootLookup = New-UniqueRootLookup $Names

$Output = @()

ForEach ($Drive in $Drives) {

    If ($Drive.BusType -Eq "NVMe") {
        $SerialNumber = $Drive.AdapterSerialNumber
        $Type = $Drive.BusType
    }
    Else { # SATA, SAS
        $SerialNumber = $Drive.SerialNumber
        $Type = $Drive.MediaType
    }

    If ($Drive.Usage -Eq "Journal") {
        $Size = $Drive.Size | ConvertTo-PrettyCapacity
        $Used = "-"
        $Percent = "-"
    }
    Else {
        $Size = $Drive.Size | ConvertTo-PrettyCapacity
        $Used = $Drive.VirtualDiskFootprint | ConvertTo-PrettyCapacity
        $Percent = ConvertTo-PrettyPercentage $Drive.VirtualDiskFootprint $Drive.Size
    }

    $NodeObj = $Drive | Get-StorageNode -PhysicallyConnected
    If ($NodeObj -Ne $Null) {
        $Node = $UniqueRootLookup[$NodeObj.Name]
    }
    Else {
        $Node = "-"
    }

    # Pack

    $Output += [PSCustomObject]@{
        "SerialNumber" = $SerialNumber
        "Type" = $Type
        "Node" = $Node
        "Size" = $Size
        "Used" = $Used
        "Percent" = $Percent
    }
}

$Output | Sort Used, Node | FT
#endregion Show-PrettyPool.ps1


#########################################################################################

#########################################################################################


 #Show-PrettyPool.ps1 - (https://blogs.technet.microsoft.com/filecab/2016/08/29/deep-dive-volumes-in-spaces-direct/)
#region Show-PrettyVolume.ps1

# Written by Cosmos Darwin, PM
# Copyright (C) 2016 Microsoft Corporation
# MIT License
# 8/2016

Function ConvertTo-PrettyCapacity {

    Param (
        [Parameter(
            Mandatory=$True,
            ValueFromPipeline=$True
            )
        ]
    [Int64]$Bytes,
    [Int64]$RoundTo = 0 # Default
    )

    If ($Bytes -Gt 0) {
        $Base = 1024 # To Match PowerShell
        $Labels = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") # Blame Snover
        $Order = [Math]::Floor( [Math]::Log($Bytes, $Base) )
        $Rounded = [Math]::Round($Bytes/( [Math]::Pow($Base, $Order) ), $RoundTo)
        [String]($Rounded) + $Labels[$Order]
    }
    Else {
        0
    }
    Return
}


Function ConvertTo-PrettyPercentage {

    Param (
        [Parameter(Mandatory=$True)]
            [Int64]$Numerator,
        [Parameter(Mandatory=$True)]
            [Int64]$Denominator,
        [Int64]$RoundTo = 0 # Default
    )

    If ($Denominator -Ne 0) { # Cannot Divide by Zero
        $Fraction = $Numerator/$Denominator
        $Percentage = $Fraction * 100
        $Rounded = [Math]::Round($Percentage, $RoundTo)
        [String]($Rounded) + "%"
    }
    Else {
        0
    }
    Return
}

### SCRIPT... ###

$Output = @()

# Query Cluster Shared Volumes
$Volumes = Get-StorageSubSystem Cluster* | Get-Volume | ? FileSystem -Eq "CSVFS"

ForEach ($Volume in $Volumes) {

    # Get MSFT_Volume Properties
    $Label = $Volume.FileSystemLabel
    $Capacity = $Volume.Size | ConvertTo-PrettyCapacity
    $Used = ConvertTo-PrettyPercentage ($Volume.Size - $Volume.SizeRemaining) $Volume.Size

    If ($Volume.FileSystemType -Like "*ReFS") {
        $Filesystem = "ReFS"
    }
    ElseIf ($Volume.FileSystemType -Like "*NTFS") {
        $Filesystem = "NTFS"
    }

    # Follow Associations
    $Partition   = $Volume    | Get-Partition
    $Disk        = $Partition | Get-Disk
    $VirtualDisk = $Disk      | Get-VirtualDisk

    # Get MSFT_VirtualDisk Properties
    $Footprint = $VirtualDisk.FootprintOnPool | ConvertTo-PrettyCapacity
    $Efficiency = ConvertTo-PrettyPercentage $VirtualDisk.Size $VirtualDisk.FootprintOnPool

    # Follow Associations
    $Tiers = $VirtualDisk | Get-StorageTier

    # Get MSFT_VirtualDisk or MSFT_StorageTier Properties...

    If ($Tiers.Length -Lt 2) {

        If ($Tiers.Length -Eq 0) {
            $ReadFrom = $VirtualDisk # No Tiers
        }
        Else {
            $ReadFrom = $Tiers[0] # First/Only Tier
        }

        If ($ReadFrom.ResiliencySettingName -Eq "Mirror") {
            # Mirror
            If ($ReadFrom.PhysicalDiskRedundancy -Eq 1) { $Resiliency = "2-Way Mirror" }
            If ($ReadFrom.PhysicalDiskRedundancy -Eq 2) { $Resiliency = "3-Way Mirror" }
            $SizeMirror = $ReadFrom.Size | ConvertTo-PrettyCapacity
            $SizeParity = [string](0)
        }
        ElseIf ($ReadFrom.ResiliencySettingName -Eq "Parity") {
            # Parity
            If ($ReadFrom.PhysicalDiskRedundancy -Eq 1) { $Resiliency = "Single Parity" }
            If ($ReadFrom.PhysicalDiskRedundancy -Eq 2) { $Resiliency = "Dual Parity" }
            $SizeParity = $ReadFrom.Size | ConvertTo-PrettyCapacity
            $SizeMirror = [string](0)
        }
        Else {
            Write-Host -ForegroundColor Red "What have you done?!"
        }
    }

    ElseIf ($Tiers.Length -Eq 2) { # Two Tiers

        # Mixed / Multi- / Hybrid
        $Resiliency = "Mix"

        ForEach ($Tier in $Tiers) {
            If ($Tier.ResiliencySettingName -Eq "Mirror") {
                # Mirror Tier
                $SizeMirror = $Tier.Size | ConvertTo-PrettyCapacity
                If ($Tier.PhysicalDiskRedundancy -Eq 1) { $Resiliency += " (2-Way" }
                If ($Tier.PhysicalDiskRedundancy -Eq 2) { $Resiliency += " (3-Way" }
            }
        }
        ForEach ($Tier in $Tiers) {
            If ($Tier.ResiliencySettingName -Eq "Parity") {
                # Parity Tier
                $SizeParity = $Tier.Size | ConvertTo-PrettyCapacity
                If ($Tier.PhysicalDiskRedundancy -Eq 1) { $Resiliency += " + Single)" }
                If ($Tier.PhysicalDiskRedundancy -Eq 2) { $Resiliency += " + Dual)" }
            }
        }
    }

    Else {
        Write-Host -ForegroundColor Red "What have you done?!"
    }

    # Pack

    $Output += [PSCustomObject]@{
        "Volume" = $Label
        "Filesystem" = $Filesystem
        "Capacity" = $Capacity
        "Used" = $Used
        "Resiliency" = $Resiliency
        "Size (Mirror)" = $SizeMirror
        "Size (Parity)" = $SizeParity
        "Footprint" = $Footprint
        "Efficiency" = $Efficiency
    }
}

$Output | Sort Efficiency, Volume | FT

#endregion Show-PrettyVolume.ps1




#########################################################################################


Comments

Post a Comment