SocketException when processing large amount of data in a single run about cxanalytix

[benjaminstokes]

Checkmarx version is 8.9 HF 0. This rather long script is what I used to generate the test data set i'm working with

# Create projects with the Checkmarx CLI
$cxUser="checkmarx"
$cxPass="redacted"
$cxServer="localhost"


[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072;
$progressPreference = 'silentlyContinue'

# Install the Checkmarx CLI
md -force c:\programdata\checkmarx\cli
$cli_url="https://download.checkmarx.com/9.0.0/Plugins/CxConsolePlugin-2020.2.18.zip"
$cli_zip=$cli_url.Substring($cli_url.LastIndexOf("/") + 1)
invoke-webrequest -uri "${cli_url}" -outfile "C:\programdata\checkmarx\cli\${cli_zip}"
cd "c:\programdata\checkmarx\cli\"
Expand-Archive "c:\programdata\checkmarx\cli\${cli_zip}" -DestinationPath "c:\programdata\checkmarx\cli\"


cd c:\programdata\checkmarx\cli\



Class CxArmPolicyManagementApiClient {
  [string] $username 
  [string] $password 
  [string] $url
  [string] $token 
  [string] $apiscope = "sast_rest_api cxarm_api"
  [string] $clientid = "resource_owner_client"
  [string] $clientsecret = "014DF517-39D1-4453-B7B3-9930C563627C"

  CxArmPolicyManagementApiClient([string] $username, [string] $password, [string] $url) {
    $this.username = $username
    $this.password = $password
    $this.url = $url
  }

  Login() {
    $body = @{
        username = $this.username
        password = $this.password
        grant_type = "password"
        scope = $this.apiscope
        client_id = $this.clientid
        client_secret = $this.clientsecret
    }
    try {
        $response = Invoke-RestMethod -uri "$($this.url)/cxrestapi/auth/identity/connect/token" -method post -body $body -contenttype 'application/x-www-form-urlencoded' -UseBasicParsing
    } catch {
        Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
        Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
        throw "Cannot Get OAuth2 Token"
    }

    $this.token = "$($response.token_type) $($response.access_token)"
  }

  [PSObject] GET([string] $url) {
    $headers = @{
        Authorization = $this.token
        Accept = "application/json;v=1.0"
    }
    try {
        $response = Invoke-RestMethod -uri "$($this.url)${url}" -method get -headers $headers -UseBasicParsing
        return $response
    } catch {
        Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
        Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
        throw "An error has occured on GET $url"
    }
  }

  [PSObject] POST([string] $url, $body) {
    $body_json = ($body | ConvertTo-Json -Depth 10)
    $headers = @{
        Authorization = $this.token       
    }
    try {
        $response = Invoke-RestMethod -uri "$($this.url)${url}" -method post -headers $headers -UseBasicParsing -Body $body_json -ContentType "application/json;v=1.0"
        return $response
    } catch {
        Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
        Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
        throw "An error has occured on POST $url with $body_json"
    }
  }

  [PSObject] PUTliteral([string] $url, $body) {
    $body_json = $body
    $headers = @{
        Authorization = $this.token       
    }
    try {
        $response = Invoke-RestMethod -uri "$($this.url)${url}" -method put -headers $headers -UseBasicParsing -Body $body_json -ContentType "application/json;v=1.0"
        return $response
    } catch {
        Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
        Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
        throw "An error has occured on PUT $url with $body_json"
    }
  }

  [PSObject] PUT([string] $url, $body) {
    $body_json = ($body | ConvertTo-Json -Depth 10)
    $headers = @{
        Authorization = $this.token       
    }
    try {
        $response = Invoke-RestMethod -uri "$($this.url)${url}" -method put -headers $headers -UseBasicParsing -Body $body_json -ContentType "application/json;v=1.0"
        return $response
    } catch {
        Write-Host "StatusCode:" $_.Exception.Response.StatusCode.value__
        Write-Host "StatusDescription:" $_.Exception.Response.StatusDescription
        throw "An error has occured on PUT $url with $body_json"
    }
  }

  <######################
  #  Projects
  #######################>
  [PSObject] GetProjects() {
    return $this.GET("/cxarm/policymanager/projects")
  }
  [PSObject] GetPoliciesByProjectId($id) {
    return $this.GET("/cxarm/policymanager/projects/${id}/policies")
  }
  [PSObject] GetViolationsByProjectId($id) {
    return $this.GET("/cxarm/policymanager/projects/${id}/violations?provider=SAST")
  }
  [PSObject] UpdatePoliciesByProjectId ($projectId, $policyId) {
    $policies = "[ { `"policyId`": $policyId } ]"
    return $this.PUTliteral("/cxarm/policymanager/projects/${projectId}/policies", $policies)
  }
  
  <######################
  #  Policies
  #######################>
  [PSObject] GetPolicies() {
    return $this.GET("/cxarm/policymanager/policies")
  }
  [PSObject] FindPolicyIdByName([string] $name) {
    $policies = $this.GetPolicies()
    $policyId = $policies | where { $_.name -eq $name } | select -ExpandProperty id
    return $policyId
  }
  [PSObject] GetPoliciesById($id) {
    return $this.GET("/cxarm/policymanager/policies/${id}")
  }
  [PSObject] GetAssignedProjectsByPoliciyId($id) {
    return $this.GET("/cxarm/policymanager/policies/${id}/projects")
  }
  [PSObject] CreatePolicy ($policy) {
    return $this.POST("/cxarm/policymanager/policies", $policy)
  }



    
  <######################
  #  Rules
  #######################>
  [PSObject] GetRulesByPolicyId($id) {
    return $this.GET("/cxarm/policymanager/policies/${id}/rules")
  }


  <######################
  #  SAST - Projects
  #######################>
  [PSObject] GetSastProjects() {
    return $this.GET("/cxrestapi/projects")
  }
  [PSObject] FindProjectByName([string] $name) {
    $projects = $this.GetSastProjects()
    $projectId = $projects | where { $_.name -eq $name } | select -ExpandProperty id
    return $projectId
  }
  [PSObject] CreateProject($project) {
    return $this.POST("/cxrestapi/projects", $project)
  }

  [PSObject] UpdateProject($project, $id) {
    return $this.PUT("/cxrestapi/projects/${id}", $project)
  }

  
  <######################
  #  SAST - Teams
  #######################>

  [PSObject] GetTeams() {
    return $this.GET("/cxrestapi/auth/teams")
  }

  [string] FindTeamIdByName([string] $teamName) {
    $teams = $this.GetTeams()
    $teamid = $teams | where { $_.fullname -eq $teamName } | select -ExpandProperty id
    return $teamid
  }



  <######################
  #  SAST - Custom Fields
  #######################>
  
  [PSObject] GetCustomFields() {
    return $this.GET("/cxrestapi/customFields")
  }

  <######################
  #  SAST - Presets
  #######################>
  
  [PSObject] GetPresets() {
    return $this.GET("/cxrestapi/sast/presets")
  }
  [PSObject] GetPresetById($id) {
    return $this.GET("/cxrestapi/sast/presets/${id}")
  }
  [PSObject] FindPresetIdByName([string] $name) {
    $presets = $this.GetPresets()
    $presetId = $presets | where { $_.name -eq $name } | select -ExpandProperty id
    return $presetId
  }

  <######################
  #  SAST - Engine Configurations
  #######################>
  
  [PSObject] GetEngineConfigurations() {
    return $this.GET("/cxrestapi/sast/engineConfigurations")
  }
  [PSObject] GetEngineConfigurationById($id) {
    return $this.GET("/cxrestapi/sast/engineConfigurations/${id}")
  }
  [PSObject] FindEngineConfigurationIdByName([string] $name) {
    $engineConfigurations = $this.GetEngineConfigurations()
    $engineConfigurationId = $engineConfigurations | where { $_.name -eq $name } | select -ExpandProperty id
    return $engineConfigurationId
  }


  <######################
  #  SAST - Scans
  #######################>
  [PSObject] CreateScanSettings($settings) {
    return $this.POST("/cxrestapi/sast/scanSettings", $settings)
  }
}


Class CxWebServiceClient {
  [string] $username 
  [string] $password 
  [string] $url
  [string] $SessionId 
  $proxy

  CxWebServiceClient([string] $username, [string] $password, [string] $url) {
    $this.username = $username
    $this.password = $password
    $this.url = $url
    $this.proxy = New-WebServiceProxy -Uri "$($this.url)/CxWebInterface/Portal/CxWebService.asmx?wsdl"
  }

  Login() {
   $proxyType = $this.proxy.GetType().Namespace
   $credentials = new-object ("$proxyType.Credentials")
   $credentials.User = $this.username
   $credentials.Pass = $this.password
   $res = $this.proxy.Login($credentials, 1033) 
   $this.SessionId = $res.SessionId
  }

  [PSObject] SaveCustomFields($fields) {
    $customfields = @()
    $fields | ForEach-Object {
        $proxyType = $this.proxy.GetType().Namespace
        $customfield = New-Object ("$proxyType.CxWSCustomField")
        $customfield.Id = 0         # Not a mistake, really zero
        $customfield.Name = $_
        $customfields+=$customfield
    }

    $res = $this.proxy.SaveCustomFields($this.sessionId, $customfields)
    if($res.IsSuccesfull){
        Write-Host "Custom fields have been created" -ForegroundColor Green
    } else{
        Write-Host $res.ErrorMessage -ForegroundColor Red
    }

    return $res
  }

  [PSObject] CreateServiceProvider($name) {
    $res = $this.proxy.CreateNewServiceProvider($this.SessionId, $name, 1, 1, 1, 1, $null)
    if ($res.IsSuccesfull) { 
        Write-Host "Created Service Provider" -ForegroundColor Green
    } else { 
        Write-Host "Error creating service provider" -ForegroundColor Red
    }
    return $res
  }

  [PSObject] CreateCompany($name, $parentId) {
    $res = $this.proxy.CreateNewCompany($this.SessionId, $parentId, $name, 1, 1, 1, 1, $null)
    if ($res.IsSuccesfull) { 
        Write-Host "Created Company" -ForegroundColor Green
    } else { 
        Write-Host "Error creating company" -ForegroundColor Red
    }
    return $res
  }

  [PSObject] CreateTeam($name, $parentId) {
    $res = $this.proxy.CreateNewTeam($this.SessionId, $parentId, $name, $null)
    if ($res.IsSuccesfull) { 
        Write-Host "Created Team" -ForegroundColor Green
    } else { 
        Write-Host "Error creating team" -ForegroundColor Red
    }
    return $res
  }
}




$projects = @(
"https://github.com/appsecco/dvna",
"https://github.com/WebGoat/WebGoat",
"https://github.com/OWASP/NodeGoat"
"https://github.com/naudio/NAudio",
"https://github.com/nopSolutions/nopCommerce",
"https://github.com/lukencode/FluentEmail",
"https://github.com/ethicalhack3r/DVWA",
"https://github.com/notepad-plus-plus/notepad-plus-plus",
"https://github.com/appsecco/dvja",
"https://github.com/CSPF-Founder/JavaVulnerableLab",
"https://github.com/nvisium-jack-mannino/OWASP-GoatDroid-Project",
"https://github.com/interference-security/DVWS",
"https://github.com/snoopysecurity/dvws-node",
"https://github.com/htbridge/pivaa",
"https://github.com/payatu/diva-android",
"https://github.com/CSPF-Founder/DodoVulnerableBank",
"https://github.com/dineshshetty/Android-InsecureBankv2",
"https://github.com/dan7800/VulnerableAndroidAppOracle",
"https://github.com/CyberScions/Digitalbank",
"https://github.com/logicalhacking/DVHMA",
"https://github.com/bkimminich/juice-shop",
"https://github.com/prateek147/DVIA",
"https://github.com/prateek147/DVIA-v2",
"https://github.com/hclproducts/AltoroJ"
)

[CxArmPolicyManagementApiClient] $cxrest = [CxArmPolicyManagementApiClient]::New($cxUser, $cxPass, $cxServer)
$cxrest.Login()

[CxWebServiceClient] $cxWs = [CxWebServiceClient]::New($cxUser, $cxPass, "http://${cxServer}")
$cxWs.Login()



foreach ($url in $projects) {
  $projectName = $url.Substring($url.LastIndexOf("/") + 1)
  $CompanyId = $cxrest.FindTeamIdByName("\CxServer\serviceprovider\company\") 
  $existingTeam = $cxrest.FindTeamIdByName("\CxServer\serviceprovider\company\${projectName}\")
  if ($null -ne $existingTeam) {
    Write-Host "SKIPPED: $team already exists" -ForegroundColor Yellow
  } else {
      Write-Host "Created \CxServer\serviceprovider\company\${projectName}\" -ForegroundColor Green
      $res = $cxWs.CreateTeam("\CxServer\serviceprovider\company\${projectName}\", $CompanyId)
  }
}




foreach ($url in $projects) {
  $projectName = $url.Substring($url.LastIndexOf("/") + 1)
  .\runCxConsole.cmd AsyncScan -v -projectName "CxServer\serviceprovider\company\${projectName}\${projectName}_sandbox" -CxServer "${cxServer}"  -CxUser ${cxUser} -CxPassword ${cxPass} -LocationType GIT -LocationURL "$url" -LocationBranch "refs/heads/master" -ForceScan
}

foreach ($url in $projects) {
  $projectName = $url.Substring($url.LastIndexOf("/") + 1)
  .\runCxConsole.cmd AsyncScan -v -projectName "CxServer\serviceprovider\company\${projectName}_policy" -CxServer "${cxServer}"  -CxUser ${cxUser} -CxPassword ${cxPass} -LocationType GIT -LocationURL "$url" -LocationBranch "refs/heads/master" -ForceScan
}

[benjaminstokes]

The root cause appears to be that so many HttpClients are created that an exception occurs. I have created a rough patch that reuses a static HttpClient with or without SSL verification enabled however the pattern of the using statements in the CxRestClient namespace causes the static HttpClient to be disposed of.

When ConcurrentThreads is > 1 this patch causes multiple exceptions to be thrown which I think indicate HttpClient is not threadsafe when used as a static (duh?).

PS C:\programdata\checkmarx\cxanalytix\artifacts\Release> dotnet CxAnalytixCLI.dll
[2020-08-12 01:58:26,968]  INFO [1] [CxAnalytixCLI.Program] (?:?) - Start
[2020-08-12 01:58:26,983]  INFO [1] [CxAnalytixCLI.Program] (?:?) - CWD: C:\programdata\checkmarx\cxanalytix\artifacts\Release
[2020-08-12 01:58:27,359]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoDBOutFactory] (?:?) - 2 calculated shard keys have been defined.
[2020-08-12 01:58:27,764]  WARN [1] [CxAnalytix.Out.MongoDBOutput.MongoDBOutFactory] (?:?) - Database CxAnalytix does not exist, it will be created.
[2020-08-12 01:58:27,784]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SAST_Scan_Detail
[2020-08-12 01:58:27,817]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SAST_Scan_Summary
[2020-08-12 01:58:27,827]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SCA_Scan_Summary
[2020-08-12 01:58:27,838]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SCA_Scan_Detail
[2020-08-12 01:58:27,847]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_Project_Info
[2020-08-12 01:58:27,856]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_Policy_Violations
[2020-08-12 01:58:27,875]  WARN [1] [CxAnalytix.TransformLogic.Transformer] (?:?) - Policy data is not available.
System.InvalidOperationException: Unable to retrieve policies.
   at CxRestClient.CxMnoPolicies.GetAllPolicies(CxRestContext ctx, CancellationToken token) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxMnoPolicies.cs:line 124
   at CxAnalytix.TransformLogic.Transformer..ctor(CxRestContext ctx, CancellationToken token, String previousStatePath) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 260
[2020-08-12 01:58:28,198]  INFO [1] [CxAnalytix.TransformLogic.ProjectResolver] (?:?) - 48 projects are targets for check for new scans. Since last scan: 0 projects removed, 48 new projects.
[2020-08-12 01:58:28,591]  INFO [1] [CxAnalytix.TransformLogic.ScanResolver] (?:?) - Resolved 52 scans to check in 48 projects since 8/12/2020 1:58:27 AM.
[2020-08-12 01:58:29,720]  WARN [1] [CxAnalytix.TransformLogic.Transformer] (?:?) - Error attempting to retrieve the SAST XML report for 1000005 in project 6: WebGoat_sandbox.
System.InvalidOperationException: Collection was modified; enumeration operation may not execute.
   at System.Collections.Generic.Dictionary`2.Enumerator.MoveNext()
   at System.Net.Http.Headers.HttpHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.Headers.HttpRequestHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.HttpClient.PrepareRequestMessage(HttpRequestMessage request)
   at System.Net.Http.HttpClient.SendAsync(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.GetAsync(Uri requestUri, HttpCompletionOption completionOption, CancellationToken cancellationToken)
   at CxRestClient.CxSastScanReportGenStatus.GetReportGenerationStatus(CxRestContext ctx, CancellationToken token, String reportId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastScanReportGenStatus.cs:line 44
   at CxRestClient.CxSastXmlReport.GetXmlReport(CxRestContext ctx, CancellationToken token, String scanId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastXmlReport.cs:line 33
   at CxAnalytix.TransformLogic.Transformer.SastReportOutput(ScanDescriptor scan, Transformer inst) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 68
[2020-08-12 01:58:29,915]  WARN [6] [CxAnalytix.TransformLogic.Transformer] (?:?) - Error attempting to retrieve the SAST XML report for 1000032 in project 29: nopCommerce_policy.
System.InvalidOperationException: Collection was modified; enumeration operation may not execute.
   at System.Collections.Generic.Dictionary`2.Enumerator.MoveNext()
   at System.Net.Http.Headers.HttpHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.Headers.HttpRequestHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.HttpClient.PrepareRequestMessage(HttpRequestMessage request)
   at System.Net.Http.HttpClient.SendAsync(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.GetAsync(Uri requestUri, HttpCompletionOption completionOption, CancellationToken cancellationToken)
   at CxRestClient.CxSastScanReportGenStatus.GetReportGenerationStatus(CxRestContext ctx, CancellationToken token, String reportId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastScanReportGenStatus.cs:line 44
   at CxRestClient.CxSastXmlReport.GetXmlReport(CxRestContext ctx, CancellationToken token, String scanId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastXmlReport.cs:line 33
   at CxAnalytix.TransformLogic.Transformer.SastReportOutput(ScanDescriptor scan, Transformer inst) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 68
[2020-08-12 01:58:30,128]  WARN [1] [CxAnalytix.TransformLogic.Transformer] (?:?) - Error attempting to retrieve the SAST XML report for 1000050 in project 47: DVIA-v2_policy.
System.ArgumentException: An item with the same key has already been added. Key: System.Net.Http.Headers.HeaderDescriptor
   at System.Collections.Generic.Dictionary`2.TryInsert(TKey key, TValue value, InsertionBehavior behavior)
   at System.Collections.Generic.Dictionary`2.Add(TKey key, TValue value)
   at System.Net.Http.Headers.HttpHeaders.GetOrCreateHeaderInfo(HeaderDescriptor descriptor, Boolean parseRawValues)
   at System.Net.Http.Headers.HttpHeaderValueCollection`1.Add(T item)
   at CxRestClient.CxRestContext.ClientFactory.CreateGenericClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 80
   at CxRestClient.CxRestContext.ClientFactory.CreateSastClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 54
atus.cs:line 44
   at CxRestClient.CxSastXmlReport.GetXmlReport(CxRestContext ctx, CancellationToken token, String scanId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastXmlReport.cs:line 33
   at CxAnalytix.TransformLogic.Transformer.SastReportOutput(ScanDescriptor scan, Transformer inst) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 68
[2020-08-12 01:58:30,832]  WARN [1] [CxAnalytix.TransformLogic.Transformer] (?:?) - Error attempting to retrieve the SAST XML report for 1000045 in project 42: VulnerableAndroidAppOracle_policy.
System.ArgumentException: An item with the same key has already been added. Key: System.Net.Http.Headers.HeaderDescriptor
   at System.Collections.Generic.Dictionary`2.TryInsert(TKey key, TValue value, InsertionBehavior behavior)
   at System.Collections.Generic.Dictionary`2.Add(TKey key, TValue value)
   at System.Net.Http.Headers.HttpHeaders.GetOrCreateHeaderInfo(HeaderDescriptor descriptor, Boolean parseRawValues)
   at System.Net.Http.Headers.HttpHeaderValueCollection`1.Add(T item)
   at CxRestClient.CxRestContext.ClientFactory.CreateGenericClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 80
   at CxRestClient.CxRestContext.ClientFactory.CreateSastClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 54
atus.cs:line 44
   at CxRestClient.CxSastXmlReport.GetXmlReport(CxRestContext ctx, CancellationToken token, String scanId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastXmlReport.cs:line 33
   at CxAnalytix.TransformLogic.Transformer.SastReportOutput(ScanDescriptor scan, Transformer inst) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 68
[2020-08-12 01:58:33,723]  WARN [1] [CxAnalytix.TransformLogic.Transformer] (?:?) - Error attempting to retrieve the SAST XML report for 1000051 in project 48: AltoroJ_policy.
System.ArgumentException: An item with the same key has already been added. Key: System.Net.Http.Headers.HeaderDescriptor
   at System.Collections.Generic.Dictionary`2.TryInsert(TKey key, TValue value, InsertionBehavior behavior)
   at System.Collections.Generic.Dictionary`2.Add(TKey key, TValue value)
   at System.Net.Http.Headers.HttpHeaders.GetOrCreateHeaderInfo(HeaderDescriptor descriptor, Boolean parseRawValues)
   at System.Net.Http.Headers.HttpHeaderValueCollection`1.Add(T item)
   at CxRestClient.CxRestContext.ClientFactory.CreateGenericClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 80
   at CxRestClient.CxRestContext.ClientFactory.CreateSastClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 54
   at CxRestClient.CxSastScanReportGenStatus.GetReportGenerationStatus(CxRestContext ctx, CancellationToken token, String reportId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastScanReportGenSt
atus.cs:line 44
   at CxRestClient.CxSastXmlReport.GetXmlReport(CxRestContext ctx, CancellationToken token, String scanId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastXmlReport.cs:line 33
   at CxAnalytix.TransformLogic.Transformer.SastReportOutput(ScanDescriptor scan, Transformer inst) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 68
[2020-08-12 01:58:35,991]  WARN [13] [CxAnalytix.TransformLogic.Transformer] (?:?) - Error attempting to retrieve the SAST XML report for 1000048 in project 45: juice-shop_policy.
System.InvalidOperationException: Operations that change non-concurrent collections must have exclusive access. A concurrent update was performed on this collection and corrupted its state. The collection'
s state is no longer correct.
   at System.Collections.Generic.Dictionary`2.FindEntry(TKey key)
   at System.Collections.Generic.Dictionary`2.TryGetValue(TKey key, TValue& value)
   at System.Net.Http.Headers.HttpHeaders.GetOrCreateHeaderInfo(HeaderDescriptor descriptor, Boolean parseRawValues)
   at System.Net.Http.Headers.HttpHeaderValueCollection`1.Add(T item)
   at CxRestClient.CxRestContext.ClientFactory.CreateGenericClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 80
   at CxRestClient.CxRestContext.ClientFactory.CreateSastClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 54
   at CxRestClient.CxSastScanReportGenStatus.GetReportGenerationStatus(CxRestContext ctx, CancellationToken token, String reportId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastScanReportGenSt
atus.cs:line 44
   at CxRestClient.CxSastXmlReport.GetXmlReport(CxRestContext ctx, CancellationToken token, String scanId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastXmlReport.cs:line 33
   at CxAnalytix.TransformLogic.Transformer.SastReportOutput(ScanDescriptor scan, Transformer inst) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 68
[2020-08-12 01:58:35,991]  WARN [1] [CxAnalytix.TransformLogic.Transformer] (?:?) - Error attempting to retrieve the SAST XML report for 1000049 in project 46: DVIA_policy.
System.InvalidOperationException: Operations that change non-concurrent collections must have exclusive access. A concurrent update was performed on this collection and corrupted its state. The collection'
s state is no longer correct.
   at System.Collections.Generic.Dictionary`2.FindEntry(TKey key)
   at System.Collections.Generic.Dictionary`2.TryGetValue(TKey key, TValue& value)
   at System.Net.Http.Headers.HttpHeaders.GetOrCreateHeaderInfo(HeaderDescriptor descriptor, Boolean parseRawValues)
   at System.Net.Http.Headers.HttpHeaderValueCollection`1.Add(T item)
   at CxRestClient.CxRestContext.ClientFactory.CreateGenericClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 80
   at CxRestClient.CxRestContext.ClientFactory.CreateSastClient() in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxRestContext.cs:line 54
   at CxRestClient.CxSastScanReportGenStatus.GetReportGenerationStatus(CxRestContext ctx, CancellationToken token, String reportId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastScanReportGenSt
atus.cs:line 44
   at CxRestClient.CxSastXmlReport.GetXmlReport(CxRestContext ctx, CancellationToken token, String scanId) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxSastXmlReport.cs:line 33
   at CxAnalytix.TransformLogic.Transformer.SastReportOutput(ScanDescriptor scan, Transformer inst) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 68
PS C:\programdata\checkmarx\cxanalytix\artifacts\Release> ^C
PS C:\programdata\checkmarx\cxanalytix\artifacts\Release> ^C
PS C:\programdata\checkmarx\cxanalytix\artifacts\Release> ^C
PS C:\programdata\checkmarx\cxanalytix\artifacts\Release> ^C

If I set concurrent threads to 1 the program runs successfully without error. Note in this example data set (52 projects w/ 48 scans) the program makes 34807 calls to MakeClient (presumably, to make a API call) which would represent an HttpClient instantiation of not using a static. Perhaps this volume of clients is what would be consuming so many outbound ports causing the original SocketException error message reported.

PS C:\programdata\checkmarx\cxanalytix\artifacts\Release> dotnet CxAnalytixCLI.dll
[2020-08-12 02:01:59,315]  INFO [1] [CxAnalytixCLI.Program] (?:?) - Start
[2020-08-12 02:01:59,327]  INFO [1] [CxAnalytixCLI.Program] (?:?) - CWD: C:\programdata\checkmarx\cxanalytix\artifacts\Release
[2020-08-12 02:01:59,646]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoDBOutFactory] (?:?) - 2 calculated shard keys have been defined.
[2020-08-12 02:02:00,074]  WARN [1] [CxAnalytix.TransformLogic.Transformer] (?:?) - Policy data is not available.
System.InvalidOperationException: Unable to retrieve policies.
   at CxRestClient.CxMnoPolicies.GetAllPolicies(CxRestContext ctx, CancellationToken token) in c:\programdata\checkmarx\CxAnalytix\CxRestClient\CxMnoPolicies.cs:line 124
   at CxAnalytix.TransformLogic.Transformer..ctor(CxRestContext ctx, CancellationToken token, String previousStatePath) in c:\programdata\checkmarx\CxAnalytix\TransformLogic\Transformer.cs:line 260
[2020-08-12 02:02:00,377]  INFO [1] [CxAnalytix.TransformLogic.ProjectResolver] (?:?) - 48 projects are targets for check for new scans. Since last scan: 0 projects removed, 48 new projects.
[2020-08-12 02:02:00,776]  INFO [1] [CxAnalytix.TransformLogic.ScanResolver] (?:?) - Resolved 52 scans to check in 48 projects since 8/12/2020 2:02:00 AM.
[2020-08-12 02:04:38,565]  INFO [1] [CxAnalytixCLI.Program] (?:?) - Rest client requested: 34807
[2020-08-12 02:04:38,565]  INFO [1] [CxAnalytixCLI.Program] (?:?) - End

[benjaminstokes]

A sketch of changing the HttpClient to a static that works in a single thread nature is in #24.

This still has issues with multiple concurrent threads but I think the fix for that is beyond my ability at the moment and may need more significant changes to the way clients are generated for SAST and MNO.

[nleach999]

The use of HttpClient needs to be refactored due to the static requirement. (I guess I missed that in the API docs....). Some of our GET ops require different media types (Json/XML) which isn't supported by the API on a per-request basis as far as I can tell, so that will be something I'll have to work out.

[nleach999]

Actually, I see the correct API in looking at it closer. It won't be too difficult to fix.

[benjaminstokes]

@nleach999 I tested the https://github.com/checkmarx-ts/CxAnalytix/tree/dev/various_improvements branch tonight with this issue (specifically this version which was the latest at the time of test 7e3f989).

I think these changes have resolved this issue.

I tested with the same dataset (actually a small amount of more projects/scans i've created in this lab machine since opening the issue) I was able to load them all into MongoDB and the log files without any error.

With MongoDB Output 2 concurrent threads

PS C:\programdata\checkmarx\CxAnalytix\artifacts\Release> dotnet .\CxAnalytixCLI.dll
[2020-09-01 04:59:50,832]  INFO [1] [CxAnalytixCLI.Program] (?:?) - Start
[2020-09-01 04:59:50,844]  INFO [1] [CxAnalytixCLI.Program] (?:?) - CWD: C:\programdata\checkmarx\CxAnalytix\artifacts\Release
[2020-09-01 04:59:51,329]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoDBOutFactory] (?:?) - 2 calculated shard keys have been defined.
[2020-09-01 04:59:52,609]  INFO [1] [CxAnalytix.TransformLogic.ProjectResolver] (?:?) - 50 projects are targets to check for new scans. Since last scan: 0 projects removed, 50 new projects.
[2020-09-01 04:59:53,113]  INFO [1] [CxAnalytix.TransformLogic.ScanResolver] (?:?) - Resolved 59 scans to check in 50 projects since 9/1/2020 4:59:51 AM.
[2020-09-01 05:01:44,824]  INFO [1] [CxAnalytixCLI.Program] (?:?) - End

With Log File Output 2 concurrent threads

PS C:\programdata\checkmarx\CxAnalytix\artifacts\Release> dotnet .\CxAnalytixCLI.dll
[2020-09-01 05:10:12,046]  INFO [1] [CxAnalytixCLI.Program] (?:?) - Start
[2020-09-01 05:10:12,058]  INFO [1] [CxAnalytixCLI.Program] (?:?) - CWD: C:\programdata\checkmarx\CxAnalytix\artifacts\Release
[2020-09-01 05:10:13,021]  INFO [1] [CxAnalytix.TransformLogic.ProjectResolver] (?:?) - 50 projects are targets to check for new scans. Since last scan: 0 projects removed, 50 new projects.
[2020-09-01 05:10:13,444]  INFO [1] [CxAnalytix.TransformLogic.ScanResolver] (?:?) - Resolved 59 scans to check in 50 projects since 9/1/2020 5:10:12 AM.
[2020-09-01 05:11:43,415]  INFO [1] [CxAnalytixCLI.Program] (?:?) - End

With MongoDB and 10 concurrent threads

PS C:\programdata\checkmarx\CxAnalytix\artifacts\Release> dotnet .\CxAnalytixCLI.dll
[2020-09-01 05:13:38,016]  INFO [1] [CxAnalytixCLI.Program] (?:?) - Start
[2020-09-01 05:13:38,030]  INFO [1] [CxAnalytixCLI.Program] (?:?) - CWD: C:\programdata\checkmarx\CxAnalytix\artifacts\Release
[2020-09-01 05:13:38,391]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoDBOutFactory] (?:?) - 2 calculated shard keys have been defined.
[2020-09-01 05:13:39,541]  INFO [1] [CxAnalytix.TransformLogic.ProjectResolver] (?:?) - 50 projects are targets to check for new scans. Since last scan: 0 projects removed, 50 new projects.
[2020-09-01 05:13:39,959]  INFO [1] [CxAnalytix.TransformLogic.ScanResolver] (?:?) - Resolved 59 scans to check in 50 projects since 9/1/2020 5:13:38 AM.
[2020-09-01 05:15:10,012]  INFO [1] [CxAnalytixCLI.Program] (?:?) - End

With MongoDB and 1 concurrent threads

PS C:\programdata\checkmarx\CxAnalytix\artifacts\Release> dotnet .\CxAnalytixCLI.dll
[2020-09-01 05:20:02,456]  INFO [1] [CxAnalytixCLI.Program] (?:?) - Start
[2020-09-01 05:20:02,468]  INFO [1] [CxAnalytixCLI.Program] (?:?) - CWD: C:\programdata\checkmarx\CxAnalytix\artifacts\Release
[2020-09-01 05:20:02,793]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoDBOutFactory] (?:?) - 2 calculated shard keys have been defined.
[2020-09-01 05:20:03,168]  WARN [1] [CxAnalytix.Out.MongoDBOutput.MongoDBOutFactory] (?:?) - Database CxAnalytix does not exist, it will be created.
[2020-09-01 05:20:03,184]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SAST_Scan_Detail
[2020-09-01 05:20:03,211]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SAST_Scan_Summary
[2020-09-01 05:20:03,221]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SCA_Scan_Summary
[2020-09-01 05:20:03,231]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_SCA_Scan_Detail
[2020-09-01 05:20:03,243]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_Project_Info
[2020-09-01 05:20:03,254]  INFO [1] [CxAnalytix.Out.MongoDBOutput.MongoUtil] (?:?) - Creating collection RECORD_Policy_Violations
[2020-09-01 05:20:04,005]  INFO [1] [CxAnalytix.TransformLogic.ProjectResolver] (?:?) - 50 projects are targets to check for new scans. Since last scan: 0 projects removed, 50 new projects.
[2020-09-01 05:20:04,429]  INFO [1] [CxAnalytix.TransformLogic.ScanResolver] (?:?) - Resolved 59 scans to check in 50 projects since 9/1/2020 5:20:03 AM.
[2020-09-01 05:23:06,884]  INFO [1] [CxAnalytixCLI.Program] (?:?) - End

I'm testing on a c5.2xlarge in aws. Memory usage did not seem to exceed 100mb. CPU usage was high when I increased the concurrent thread count (expected). I think its unrelated to this issue but I took some screenshots of CPU just as reference point.

CPU Utilization w/ Mongo DB and 10 concurrent scans

CPU Utilization w/ Mongo DB and 2 concurrent scans