diff --git a/.github/scripts/rows/fetch-symbols.ps1 b/.github/scripts/rows/fetch-symbols.ps1 deleted file mode 100644 index 118c4d7..0000000 --- a/.github/scripts/rows/fetch-symbols.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -param( - [Parameter(Mandatory = $true)][string]$Browser, - [Parameter(Mandatory = $true)][string]$Key, - [Parameter(Mandatory = $true)][string]$Out, - # Prints the address without fetching it. - [switch]$ResolveOnly -) - -$ErrorActionPreference = 'Stop' -# Each maker publishes its symbols in its own place. Brave keeps them gzipped. These are only for -# making rows on the build machine: a Rivet on a person's computer never downloads a PDB. -$url, $packed = switch ($Browser) { - 'Google Chrome' { "https://chromium-browser-symsrv.commondatastorage.googleapis.com/chrome.dll.pdb/$Key/chrome.dll.pdb", $false } - 'Microsoft Edge' { "https://msdl.microsoft.com/download/symbols/msedge.dll.pdb/$Key/msedge.dll.pdb", $false } - 'Brave' { "https://brave-symbols.s3.brave.com/chrome.dll.pdb/$Key/chrome.dll.pdb", $true } - default { throw "No public symbols are known for $Browser" } -} -if ($ResolveOnly) { $url; return } -$target = if ($packed) { "$Out.gz" } else { $Out } -curl.exe -fsSL --retry 3 -o $target $url -# curl exits with 22 when the server answers with an error, which for a symbol server means the PDB -# is not published yet. The caller may then make the row without it, so that case has its own type. -if ($LASTEXITCODE -eq 22) { throw [System.IO.FileNotFoundException]::new("The symbol server has no PDB at $url yet") } -if ($LASTEXITCODE -ne 0) { throw "Could not download ${url}: curl exited with $LASTEXITCODE" } -if ($packed) { - $in = [IO.File]::OpenRead($target) - $gzip = [IO.Compression.GZipStream]::new($in, [IO.Compression.CompressionMode]::Decompress) - $to = [IO.File]::Create($Out) - $gzip.CopyTo($to) - $to.Dispose(); $gzip.Dispose(); $in.Dispose() - Remove-Item $target -} diff --git a/.github/workflows/browser-rows.yml b/.github/workflows/browser-rows.yml index d0c5a42..5b0cbf5 100644 --- a/.github/workflows/browser-rows.yml +++ b/.github/workflows/browser-rows.yml @@ -1,9 +1,9 @@ # Keeps Rivet's supported-build table ahead of Chrome, Edge and Brave. # # A browser updates itself whenever its maker says, and Rivet does not hold it back. So this job -# watches what the makers publish, builds a whole row for each new build from public symbols, -# proves the row on the real browser, and publishes a signed table that reaches people without -# reinstalling Rivet. A row the generator cannot vouch for, such as the first of a new milestone, +# watches what the makers publish, builds a whole row for each new build from the verified row of +# the build before it, proves the row on the real browser, and publishes a signed table that reaches +# people without reinstalling Rivet. A row the generator cannot vouch for, such as the first of a new milestone, # becomes a pull request for a person and is not published. # # Needs on the default branch: this file. Needs in the repository's secrets: @@ -18,11 +18,16 @@ # that failed, its reason and the capture of the failing window. Each step records its reason in # `failure.txt` before it stops. # -# A maker can ship a build before its PDB reaches the symbol server. Then the row is made from the -# fingerprints of the prior row's functions, each of which must match one place in the new module, -# and every offset, slot and size is read again from the new module as for a row made from symbols. -# Such a row is marked, so once the PDB appears the job makes the row again from it and compares the -# two. A difference opens an issue. +# No browser's debug symbols are downloaded or read. A maker can ship a build long before its +# symbols appear, or never publish them. A row is made from the verified prior row, the fingerprints +# of its functions, the prior build's module and the new build's module: each function must match +# one place, or two independent methods must agree on it, and every offset, slot and size is read +# again from the new module. A feature whose values are not found is off in the row, and a part the +# window cannot do without stops the row for a person. +# +# A run started by hand with `score` makes, the same way, the rows of builds that already have a +# verified row, and compares them value by value. It is a measurement: it publishes, proposes and +# reports nothing, and fails only when a value it found differs from the verified one. name: Browser rows on: @@ -44,6 +49,10 @@ on: description: "With trial: the builds to make, as browser@version separated by semicolons, instead of those the feeds list." required: false default: "" + score: + description: "Score the generator on builds that have a verified row, as browser@prior>new separated by semicolons, or 'default' for scripts/tests/row-score.ps1's list. Nothing is published." + required: false + default: "" permissions: contents: write @@ -59,7 +68,7 @@ concurrency: env: RELEASE_TAG: builds-table - # Symbols are several gigabytes, and a run does no more than this many builds. + # Each build is a download of hundreds of megabytes, and a run does no more than this many builds. MOST_PER_RUN: "3" jobs: @@ -112,7 +121,7 @@ jobs: watch: name: Find builds with no row needs: gate - if: needs.gate.outputs.enabled == 'true' && github.event.inputs.withdraw == '' + if: needs.gate.outputs.enabled == 'true' && github.event.inputs.withdraw == '' && github.event.inputs.score == '' runs-on: windows-latest defaults: run: @@ -193,41 +202,8 @@ jobs: "::warning::$($build.browser) $($build.version) has no row and cannot be fetched yet: $_" } } - # A row made without its PDB is made again from the PDB once the symbol server has it, and - # the two are compared. Such a row may wait in develop's table or on a row branch. - $sources = @([pscustomobject]@{ source = 'develop'; file = 'rivet/runtime/windows/rows/table.json' }) - foreach ($line in @(git ls-remote --heads origin)) { - $branch = ($line -split "`t")[1] -replace '^refs/heads/', '' - if (-not ($branch -like 'feat/row-*' -or $branch -like 'feat/rows-*') -or $branch -like 'feat/row-pdb-*') { continue } - git fetch --quiet --depth 1 origin $branch - if ($LASTEXITCODE -ne 0) { continue } - $file = Join-Path $feeds ("table-" + ($branch -replace '[^A-Za-z0-9.]+', '-') + ".json") - git show "FETCH_HEAD:rivet/runtime/windows/rows/table.json" | Set-Content $file - if ($LASTEXITCODE -eq 0) { $sources += [pscustomobject]@{ source = $branch; file = $file } } - } - $open = @(gh issue list --state open --label browser-rows --limit 100 --json title | ConvertFrom-Json | ForEach-Object title) - $seen = @{} - $rechecks = foreach ($source in $sources) { - foreach ($row in @((Get-Content -Raw $source.file | ConvertFrom-Json).rows)) { - $mark = $row.made_without_pdb - if (-not $mark) { continue } - $browser = $row.frame.browser - $version = $row.frame.version - if ($seen["$browser $version"]) { continue } - $seen["$browser $version"] = $true - # A confirmation waits for review, and a difference waits on its issue. - $slug = "$($browser -replace ' ','-')-$version".ToLower() - git ls-remote --exit-code --heads origin "feat/row-pdb-$slug" | Out-Null - if ($LASTEXITCODE -eq 0) { continue } - if ($open -contains "Row differs from its PDB: $browser $version") { continue } - $url = ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key $mark.pdb_key -Out unused -ResolveOnly - curl.exe -fsSL -r 0-0 -o NUL $url - if ($LASTEXITCODE -ne 0) { "$browser $version still has no PDB."; continue } - [pscustomobject]@{ browser = $browser; version = $version; channel = $mark.channel; download = $mark.download; recheck = $source.source } - } - } $global:LASTEXITCODE = 0 - $builds = @(@($builds) + @($rechecks) | Where-Object { $_ -and $_ -isnot [string] } | Select-Object -First ([int]$env:MOST_PER_RUN)) + $builds = @(@($builds) | Where-Object { $_ -and $_ -isnot [string] } | Select-Object -First ([int]$env:MOST_PER_RUN)) # Through the pipeline: -InputObject would wrap the whole list as one element of another list. $json = if ($builds.Count) { $builds | ConvertTo-Json -Compress -AsArray } else { "[]" } "builds=$json" >> $env:GITHUB_OUTPUT @@ -263,7 +239,7 @@ jobs: name: row-tools path: tools - - name: Get the build and its symbols + - name: Get the build id: build run: | trap { Add-Content (Join-Path $env:RUNNER_TEMP failure.txt) "Get the build: $_"; break } @@ -272,114 +248,50 @@ jobs: -Channel "${{ matrix.build.channel }}" -Download "${{ matrix.build.download }}" ` -Out (Join-Path $env:RUNNER_TEMP "browser") -Output $env:GITHUB_OUTPUT - - name: Download the symbols of this build and of the row before it - id: symbols - env: - RECHECK: ${{ matrix.build.recheck }} + - name: Get the build of the row before it + id: prior run: | - trap { Add-Content (Join-Path $env:RUNNER_TEMP failure.txt) "Download the symbols: $_"; break } + trap { Add-Content (Join-Path $env:RUNNER_TEMP failure.txt) "Get the prior build: $_"; break } $browser = "${{ matrix.build.browser }}" $version = "${{ matrix.build.version }}" $temp = $env:RUNNER_TEMP $prior = Join-Path $temp "prior.json" - $tables = @("--table", "rivet/runtime/windows/rows/table.json") - $before = $version - if ($env:RECHECK) { - # The row made without its PDB, from where it is kept, and the prior row it was made from. - $stored = Join-Path $temp "stored-table.json" - if ($env:RECHECK -eq 'develop') { - Copy-Item rivet/runtime/windows/rows/table.json $stored - } else { - git fetch --quiet --depth 1 origin $env:RECHECK - if ($LASTEXITCODE -ne 0) { throw "Could not fetch $env:RECHECK" } - git show "FETCH_HEAD:rivet/runtime/windows/rows/table.json" | Set-Content $stored - if ($LASTEXITCODE -ne 0) { throw "$env:RECHECK has no table" } - } - $row = @((Get-Content -Raw $stored | ConvertFrom-Json).rows) | - Where-Object { $_.frame.browser -eq $browser -and $_.frame.version -eq $version } | Select-Object -First 1 - if (-not $row.made_without_pdb) { throw "$env:RECHECK holds no row of $browser $version made without its PDB" } - $before = $row.made_without_pdb.prior_version - $tables += @("--table", $stored) - } - ./tools/rivet-rows.exe prior $browser $before @tables | Set-Content $prior + ./tools/rivet-rows.exe prior $browser $version --table rivet/runtime/windows/rows/table.json | Set-Content $prior if ($LASTEXITCODE -ne 0) { throw "rivet-rows found no prior row for $browser $version" } - $priorRow = Get-Content -Raw $prior | ConvertFrom-Json - $priorVersion = $priorRow.frame.version - $priorKey = $null - foreach ($file in @($tables | Where-Object { $_ -ne '--table' })) { - if (-not $priorKey) { $priorKey = (Get-Content -Raw $file | ConvertFrom-Json).symbols."$browser $priorVersion" } - } - if (-not $priorKey) { throw "The table holds no symbol key for the prior row $browser $priorVersion. Add it to rows/table.json with chrome-builds pdb-key." } - $newKey = ./tools/chrome-builds.exe pdb-key "${{ steps.build.outputs.module }}" - if ($LASTEXITCODE -ne 0) { throw "chrome-builds could not read the symbol key of the new module" } - # The prior row's PDB names what the row is made from, so without it nothing is made. - ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key $priorKey -Out (Join-Path $temp "prior.pdb") + $priorVersion = (Get-Content -Raw $prior | ConvertFrom-Json).frame.version + # The prior build's own module gives what its row was made from: its function starts, + # its class names and the instructions the methods compare with the new build. + $found = Join-Path $temp "prior-build.txt" + $fetch = @{ Browser = $browser; Version = $priorVersion; Out = (Join-Path $temp "prior-browser"); Output = $found } try { - ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key ($newKey -split '/')[1] -Out (Join-Path $temp "new.pdb") - "no_pdb=false" >> $env:GITHUB_OUTPUT - } catch [System.IO.FileNotFoundException] { - if ($env:RECHECK) { throw } - "::warning::$($_.Exception.Message). The row is made from the fingerprints of the $browser $priorVersion row." - "no_pdb=true" >> $env:GITHUB_OUTPUT - # The prior build gives the fingerprints when the prior row stored none, and in any case - # tells which prior addresses are function starts, so a match elsewhere does not count. - $found = Join-Path $temp "prior-build.txt" - $fetch = @{ Browser = $browser; Version = $priorVersion; Out = (Join-Path $temp "prior-browser"); Output = $found } - try { - ./.github/scripts/rows/fetch-browser.ps1 @fetch -Channel "${{ matrix.build.channel }}" - } catch { - if ("${{ matrix.build.channel }}" -eq 'stable') { throw "The symbol server has no PDB for $browser $version, and the prior build $priorVersion could not be fetched for its fingerprints: $_" } - ./.github/scripts/rows/fetch-browser.ps1 @fetch -Channel stable - } - $priorModule = (Get-Content $found | Where-Object { $_ -like 'module=*' } | Select-Object -First 1) -replace '^module=', '' - $priorModuleKey = ./tools/chrome-builds.exe pdb-key $priorModule - if (($priorModuleKey -split '/')[1] -ne $priorKey) { throw "The fetched $browser $priorVersion is not the build the prior row was made for" } - "prior_module=$priorModule" >> $env:GITHUB_OUTPUT + ./.github/scripts/rows/fetch-browser.ps1 @fetch -Channel "${{ matrix.build.channel }}" + } catch { + if ("${{ matrix.build.channel }}" -eq 'stable') { throw "The prior build $browser $priorVersion could not be fetched: $_" } + ./.github/scripts/rows/fetch-browser.ps1 @fetch -Channel stable } + $priorModule = (Get-Content $found | Where-Object { $_ -like 'module=*' } | Select-Object -First 1) -replace '^module=', '' + "module=$priorModule" >> $env:GITHUB_OUTPUT - name: Make the row id: made env: - NO_PDB: ${{ steps.symbols.outputs.no_pdb }} - PRIOR_MODULE: ${{ steps.symbols.outputs.prior_module }} - RECHECK: ${{ matrix.build.recheck }} - CHANNEL: ${{ matrix.build.channel }} - DOWNLOAD: ${{ matrix.build.download }} + PRIOR_MODULE: ${{ steps.prior.outputs.module }} run: | trap { Add-Content (Join-Path $env:RUNNER_TEMP failure.txt) "Make the row: $_"; break } $temp = $env:RUNNER_TEMP - $browser = "${{ matrix.build.browser }}" - $version = "${{ matrix.build.version }}" - if ($env:NO_PDB -eq 'true') { - # Every function is looked for by its fingerprint, and one that does not match exactly - # one place leaves the row for review. Offsets, slots and sizes are read as below. - ./tools/chrome-builds.exe find "$temp/prior.json" "$temp/prior.pdb" $env:PRIOR_MODULE "${{ steps.build.outputs.module }}" $version $env:CHANNEL "$env:DOWNLOAD" "$temp/row.json" "$temp/report.json" - $code = $LASTEXITCODE - if ($code -ne 0 -and $code -ne 3) { throw "chrome-builds find exited with $code" } - } else { - ./tools/chrome-builds.exe port "$temp/prior.json" "$temp/prior.pdb" "$temp/new.pdb" "${{ steps.build.outputs.module }}" $version "$temp/row.json" "$temp/report.json" - $code = $LASTEXITCODE - if ($code -ne 0 -and $code -ne 3) { throw "chrome-builds port exited with $code; report.json says why" } - } - if ($env:RECHECK) { - # The PDB is out now: the row made from it must agree with the row made without it. - ./tools/chrome-builds.exe pdb-check "$temp/stored-table.json" $browser $version "$temp/row.json" | Tee-Object "$temp/differences.txt" - $check = $LASTEXITCODE - if ($check -ne 0 -and $check -ne 4) { throw "chrome-builds pdb-check exited with $check" } - "checked=$(if ($check -eq 0) { 'agree' } else { 'differ' })" >> $env:GITHUB_OUTPUT - "for_review=recheck" >> $env:GITHUB_OUTPUT - Get-Content "$temp/report.json" - exit 0 - } - # Fingerprints let a Rivet find these functions in a later build that has no row yet. A row - # whose addresses cannot all be fingerprinted is still a good row, just without them. A row - # made without its PDB that is not vouched for still holds prior addresses, which would - # fingerprint the wrong code, so it gets none. - if ($env:NO_PDB -eq 'true' -and $code -ne 0) { - "A row made without its PDB that needs review gets no fingerprints." - } else { + # Every function is looked for by its fingerprint and the methods that read the prior + # build next to the new one. Offsets, slots and sizes are read from the new module. + ./tools/chrome-builds.exe find "$temp/prior.json" $env:PRIOR_MODULE "${{ steps.build.outputs.module }}" "${{ matrix.build.version }}" "$temp/row.json" "$temp/report.json" + $code = $LASTEXITCODE + if ($code -ne 0 -and $code -ne 3) { throw "chrome-builds find exited with $code; report.json says why" } + # Fingerprints, read from this build's own module at the places the row holds, let the + # next build's row start from this one. A row that needs review may still hold a prior + # address, which would fingerprint the wrong code, so it gets none. + if ($code -eq 0) { ./tools/chrome-builds.exe fingerprint "$temp/row.json" "${{ steps.build.outputs.module }}" "$temp/row.json" if ($LASTEXITCODE -ne 0) { "The row has no fingerprints." } + } else { + "A row that needs review gets no fingerprints." } "for_review=$($code -eq 3)".ToLower() >> $env:GITHUB_OUTPUT Get-Content "$temp/report.json" @@ -398,8 +310,8 @@ jobs: Set-Content (Join-Path $temp "test-key.txt") $env:RIVET_TABLE_PUBLIC_KEY $env:LOCALAPPDATA = Join-Path $temp "local" New-Item -ItemType Directory -Force $env:LOCALAPPDATA | Out-Null - Set-Content "$temp/part.json" '{"schema":1,"sequence":0,"symbols":{},"rows":[]}' - ./tools/chrome-builds.exe table-add "$temp/part.json" "$temp/row.json" "${{ steps.build.outputs.module }}" + Set-Content "$temp/part.json" '{"schema":1,"sequence":0,"rows":[]}' + ./tools/chrome-builds.exe table-add "$temp/part.json" "$temp/row.json" ./tools/chrome-builds.exe table-sign "$temp/part.json" "$temp/test.bin" Push-Location rivet cargo build -p rivet -p rivet-runtime-windows @@ -453,60 +365,8 @@ jobs: ${{ runner.temp }}/session/features/traces/**/*.log ${{ runner.temp }}/session/features/*/*.png ${{ runner.temp }}/session/features/profiles/*/Crashpad/reports/*.dmp - ${{ runner.temp }}/differences.txt retention-days: 30 - - name: Tell a person that a row made without its PDB differs from it - if: steps.made.outputs.checked == 'differ' && !inputs.trial - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RECHECK: ${{ matrix.build.recheck }} - run: | - $temp = $env:RUNNER_TEMP - $browser = "${{ matrix.build.browser }}" - $version = "${{ matrix.build.version }}" - $reasons = Join-Path $temp "differences-reasons.txt" - Set-Content $reasons "The row in $env:RECHECK was made from fingerprints, and the row its PDB gives differs from it:" - Get-Content "$temp/differences.txt" | Add-Content $reasons - ./.github/scripts/rows/report-failure.ps1 ` - -Title "Row differs from its PDB: $browser $version" ` - -Reasons $reasons -Files "${{ steps.files.outputs.artifact-url }}" - - - name: Confirm a row made without its PDB - if: steps.made.outputs.checked == 'agree' && !inputs.trial - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RECHECK: ${{ matrix.build.recheck }} - run: | - $temp = $env:RUNNER_TEMP - $browser = "${{ matrix.build.browser }}" - $version = "${{ matrix.build.version }}" - git config user.name "rivet-browser-rows" - git config user.email "rivet-browser-rows@users.noreply.github.com" - # A row on its own branch is confirmed there. A row in develop is confirmed by a pull request. - if ($env:RECHECK -eq 'develop') { - $branch = "feat/row-pdb-$($browser -replace ' ','-')-$version".ToLower() - git switch -c $branch - } else { - $branch = $env:RECHECK - git fetch --quiet --depth 1 origin $branch - git switch -c $branch FETCH_HEAD - } - ./tools/chrome-builds.exe pdb-check rivet/runtime/windows/rows/table.json $browser $version "$temp/row.json" - if ($LASTEXITCODE -ne 0) { throw "chrome-builds pdb-check exited with $LASTEXITCODE on $branch" } - git add rivet/runtime/windows/rows/table.json - git commit -m "Fix: Confirm the row of $browser $version with its PDB" - git push origin "HEAD:$branch" - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $note = "The row of $browser $version was made from fingerprints because its PDB was not published. The PDB is out now, the row made from it agrees, and the mark is removed in run $env:GITHUB_RUN_ID." - if ($env:RECHECK -eq 'develop') { - gh pr create --base develop --head $branch --title "Fix: Confirm the row of $browser $version with its PDB" --body $note - gh workflow run test.yml --ref $branch - } else { - gh pr comment $branch --body $note - $global:LASTEXITCODE = 0 - } - - name: Tell a person that this build failed if: failure() && !inputs.trial env: @@ -530,15 +390,12 @@ jobs: git config user.name "rivet-browser-rows" git config user.email "rivet-browser-rows@users.noreply.github.com" git switch -c $branch - ./tools/chrome-builds.exe table-add rivet/runtime/windows/rows/table.json "$temp/row.json" "${{ steps.build.outputs.module }}" + ./tools/chrome-builds.exe table-add rivet/runtime/windows/rows/table.json "$temp/row.json" git add rivet/runtime/windows/rows/table.json git commit -m "Feat: Propose a row for ${{ matrix.build.browser }} ${{ matrix.build.version }}" git push -u origin $branch $report = Get-Content -Raw "$temp/report.json" | ConvertFrom-Json - $body = "The generator could not vouch for this row, so it was not published.`n`n" - if ($report.made_without_pdb) { - $body += "The symbol server had no PDB for this build, so the row was made from the fingerprints of the $($report.prior_version) row: $($report.fingerprinted.Count) of $($report.addresses) addresses matched exactly one place. When the PDB appears, the row job makes the row again from it and compares the two.`n`n" - } + $body = "The generator could not vouch for this row, so it was not published. It was made from the $($report.prior_version) row: $($report.fingerprinted.Count) of $($report.addresses) addresses matched exactly one place by fingerprint, and $($report.settled.Count) were settled by two methods.`n`n" $body += (($report.review | ForEach-Object { "- $_" }) -join "`n") if ($report.switched_off.Count -gt 0) { $body += "`n`nThese features are off in this row, because a value they need could not be read from this build:`n`n" @@ -621,3 +478,37 @@ jobs: git push -u origin $branch gh pr create --base develop --head $branch --title "Feat: Record rows published by the table job" --body "These rows are already published in the signed table and passed the row proof in run $env:GITHUB_RUN_ID. Merging adds them to the table compiled into the next Rivet." gh workflow run test.yml --ref $branch + + score: + name: Score the generator against verified rows + # A measurement: it needs no signing key, and it publishes, proposes and reports nothing. + if: github.event.inputs.score != '' + runs-on: windows-latest + permissions: + contents: read + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@v4 + + - name: Build the row tools + working-directory: rivet + run: cargo build --release -p rivet-platform-windows -p rivet-runtime-windows --bins + + - name: Make each row without symbols and compare it with its verified row + env: + SCORE: ${{ github.event.inputs.score }} + run: | + $pairs = if ($env:SCORE -eq 'default') { @() } else { @($env:SCORE -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) } + ./scripts/tests/row-score.ps1 ` + -Rows rivet/target/release/rivet-rows.exe -Generator rivet/target/release/chrome-builds.exe ` + -Table rivet/runtime/windows/rows/table.json -Pairs $pairs -Fetch ` + -Work (Join-Path $env:RUNNER_TEMP "score") + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: row-score + path: ${{ runner.temp }}/score/*.json + retention-days: 30 diff --git a/rivet/platform/windows/src/bin/chrome-builds.rs b/rivet/platform/windows/src/bin/chrome-builds.rs index af0f457..d70f005 100644 --- a/rivet/platform/windows/src/bin/chrome-builds.rs +++ b/rivet/platform/windows/src/bin/chrome-builds.rs @@ -1,25 +1,17 @@ -//! Adds a Chromium build to Rivet's supported-build table from the build's public symbols. +//! Makes and publishes rows of Rivet's supported-build table. No browser's debug symbols are read: +//! a new row comes from the verified prior row, its fingerprints, the prior module and the new one. //! -//! `chrome-builds pdb-key ` prints the symbol-server path of the module's PDB. -//! `chrome-builds extract [frame-class]` -//! resolves the layout hook's addresses with DbgHelp, checks them against the module file and -//! updates the table. The frame class is `BrowserFrameViewWin` unless the browser builds its -//! windows from a subclass of it. -//! `chrome-builds port ` -//! makes a whole row for a new build from the row of a prior build, and exits 3 when the row needs -//! a person's review. -//! `chrome-builds find ` -//! makes a candidate row for a new build whose PDB is not published, from the fingerprints of the -//! prior row: those it stored, or else those read now from the prior build's module. It marks the -//! row as made without its PDB and exits 3 when the row needs a person's review. -//! `chrome-builds pdb-check ` compares a row made -//! without its PDB with the row its PDB gives. It removes the mark when they agree and exits 4, -//! printing each difference, when they do not. +//! `chrome-builds find ` +//! makes a candidate row for a new build and exits 3 when the row needs a person's review. +//! `chrome-builds score ` +//! makes the row of a build that already has a verified row the same way, compares the two value +//! by value and writes the score of each feature. It exits 5 when a value it found is wrong. +//! `chrome-builds fingerprint ` adds the fingerprints of a row's +//! functions, read from its own module, so the next build's row can start from them. //! `chrome-builds pending ` //! prints the builds that have no row yet, most urgent first. //! `chrome-builds table-key` prints a new signing key pair. -//! `chrome-builds table-add ` puts a row into a table, with the -//! symbol key the next build's row will need. +//! `chrome-builds table-add ` puts a row into a table. //! `chrome-builds table-read ` writes out a table that verifies under //! `RIVET_TABLE_PUBLIC_KEY`. //! `chrome-builds table-merge ...` adds the rows of other tables to a table. @@ -30,35 +22,17 @@ use std::path::Path; use std::process::ExitCode; -use rivet_platform_windows::chrome_build::{self, ChromeBuild, PeFile}; -use rivet_platform_windows::row_port::{self, NewBuild}; +use rivet_platform_windows::chrome_build::{self, PeFile}; +use rivet_platform_windows::row_port::{self, PortReport}; use rivet_platform_windows::{row_watch, signed_table}; -/// How many bytes of `views::HWNDForView` the runtime compares before patching. -const PROLOGUE: usize = 15; - fn main() -> ExitCode { let arguments: Vec = std::env::args().skip(1).collect(); let result = match arguments.iter().map(String::as_str).collect::>().as_slice() { - ["pdb-key", module] => pdb_key(Path::new(module)), - ["names", pdb, needles @ ..] => names(Path::new(pdb), needles), - ["near", pdb, addresses @ ..] => near(Path::new(pdb), addresses), - ["extract", module, pdbs, browser, version, table] => extract( - Path::new(module), - Path::new(pdbs), - browser, - version, - Path::new(table), - "BrowserFrameViewWin", - ), - ["extract", module, pdbs, browser, version, table, class] => { - extract(Path::new(module), Path::new(pdbs), browser, version, Path::new(table), class) - } - ["port", prior, old_pdb, new_pdb, module, version, row, report] => { - return match port( + ["find", prior, prior_module, module, version, row, report] => { + return match find( Path::new(prior), - Path::new(old_pdb), - Path::new(new_pdb), + Path::new(prior_module), Path::new(module), version, Path::new(row), @@ -72,30 +46,16 @@ fn main() -> ExitCode { } } } - ["find", prior, old_pdb, old_module, module, version, channel, download, row, report] => { - return match find( + ["score", prior, prior_module, module, verified, out] => { + return match score( Path::new(prior), - Path::new(old_pdb), - old_module, + Path::new(prior_module), Path::new(module), - version, - channel, - download, - Path::new(row), - Path::new(report), + Path::new(verified), + Path::new(out), ) { Ok(true) => ExitCode::SUCCESS, - Ok(false) => ExitCode::from(3), - Err(error) => { - eprintln!("chrome-builds: {error}"); - ExitCode::FAILURE - } - } - } - ["pdb-check", table, browser, version, row] => { - return match pdb_check(Path::new(table), browser, version, Path::new(row)) { - Ok(true) => ExitCode::SUCCESS, - Ok(false) => ExitCode::from(4), + Ok(false) => ExitCode::from(5), Err(error) => { eprintln!("chrome-builds: {error}"); ExitCode::FAILURE @@ -114,7 +74,7 @@ fn main() -> ExitCode { println!("private {private}"); println!("public {public}"); }), - ["table-add", table, row, module] => table_add(Path::new(table), Path::new(row), Path::new(module)), + ["table-add", table, row] => table_add(Path::new(table), Path::new(row)), ["table-read", signed, out] => table_read(Path::new(signed), Path::new(out)), ["table-merge", table, parts @ ..] => table_merge(Path::new(table), parts), ["table-sign", table, signed] => table_sign(Path::new(table), Path::new(signed), None), @@ -124,7 +84,7 @@ fn main() -> ExitCode { ["table-withdraw", published, signed, builds @ ..] if !builds.is_empty() => { table_withdraw(Path::new(published), Path::new(signed), builds) } - _ => Err("usage: chrome-builds pdb-key | extract
[frame-class] | port | find | pdb-check | pending | gate [row.json] | fingerprint | release-sign ... | release-check | table-key | table-add | table-read | table-merge ... | table-sign [published-file] | table-withdraw ...".into()), + _ => Err("usage: chrome-builds find | score | pending | gate [row.json] | fingerprint | release-sign ... | release-check | table-key | table-add | table-read | table-merge ... | table-sign [published-file] | table-withdraw ...".into()), }; match result { Ok(()) => ExitCode::SUCCESS, @@ -135,246 +95,139 @@ fn main() -> ExitCode { } } -/// Prints the address and decorated name of every public symbol whose name holds one of the needles. -fn names(pdb: &Path, needles: &[&str]) -> Result<(), String> { - symbols::publics(pdb, &mut |address, name| { - if needles.iter().any(|needle| name.contains(needle)) { - println!("{address:08X} {name}"); - } - }) -} - -/// Prints, for each address in hex, the public symbol that starts at or just before it. -fn near(pdb: &Path, addresses: &[&str]) -> Result<(), String> { - let wanted: Vec = addresses - .iter() - .filter_map(|text| u32::from_str_radix(text.trim_start_matches("0x"), 16).ok()) - .collect(); - let mut best: Vec> = vec![None; wanted.len()]; - symbols::publics(pdb, &mut |address, name| { - for (slot, target) in wanted.iter().enumerate() { - if address <= *target && best[slot].as_ref().is_none_or(|(at, _)| address > *at) { - best[slot] = Some((address, name.to_owned())); - } - } - })?; - for (target, found) in wanted.iter().zip(best) { - match found { - Some((at, name)) => println!("{target:08X} is {:#x} past {at:08X} {name}", target - at), - None => println!("{target:08X} has no symbol before it"), - } - } - Ok(()) -} - fn read(path: &Path) -> Result, String> { std::fs::read(path).map_err(|error| format!("could not read {}: {error}", path.display())) } -fn port( - prior: &Path, - old_pdb: &Path, - new_pdb: &Path, +/// Makes the row of `version` from the prior row, the prior row's own module and the new module. +fn make( + prior: &serde_json::Value, + prior_module: &Path, module: &Path, version: &str, - row_path: &Path, - report_path: &Path, -) -> Result { - let prior: serde_json::Value = serde_json::from_slice(&read(prior)?) - .map_err(|error| format!("the prior row is not JSON: {error}"))?; - let bytes = read(module)?; - let pe = PeFile::parse(&bytes).ok_or("not a 64-bit PE image")?; - let mut old = Vec::new(); - symbols::publics(old_pdb, &mut |address, name| { - old.push((address, name.to_owned())) - })?; - old.sort(); - // The frame entry keeps plain numbers. - let mut targets = row_port::addresses_of(&prior["build"]); - for key in ["frame_view_vtable", "client_view_bounds", "hwnd_for_view"] { - targets.extend(prior["frame"][key].as_u64().map(|value| value as u32)); +) -> Result<(serde_json::Value, PortReport), String> { + let prior_bytes = read(prior_module)?; + let prior_pe = + PeFile::parse(&prior_bytes).ok_or("the prior module is not a 64-bit PE image")?; + // A module of another build would give fingerprints and class names of the wrong functions. + let start = prior["frame"]["hwnd_for_view"] + .as_u64() + .and_then(|address| u32::try_from(address).ok()); + let prologue = prior["frame"]["hwnd_for_view_prologue"] + .as_str() + .and_then(chrome_build::hex); + match (start, prologue) { + (Some(start), Some(prologue)) + if prior_pe.read(start, prologue.len()) == Some(prologue.as_slice()) => {} + _ => return Err("the prior module is not the build of the prior row".into()), } - let named = row_port::name_addresses(&targets, &old); - let wanted = row_port::wanted_names(&named); - drop(old); - let mut new = std::collections::HashMap::>::new(); - // `views::View::ConvertPointToScreen` is found by its own name, for a prior row without it. - let mut screen_origin = Vec::new(); - symbols::publics(new_pdb, &mut |address, name| { - if row_port::is_screen_origin_name(name) { - screen_origin.push(address); - } - let name = row_port::stable_name(name); - if wanted.contains(name) { - new.entry(name.to_owned()).or_default().push(address); + // The fingerprints the prior row stored were read from this module when the row was made. + // A row stored without them gets them from the module now. + let prints = match prior["fingerprints"].as_object() { + Some(stored) => stored.clone(), + None => { + let (prints, missing) = row_port::fingerprints_of(prior, &prior_pe); + if !missing.is_empty() { + println!( + "the prior build gives no fingerprint for {}", + missing.join(", ") + ); + } + prints } - })?; - let unnamed = row_port::unnamed_code(&prior, &named, &pe); - let names = symbols::private_names(old_pdb, &unnamed)?; - let places = symbols::private_places(new_pdb, &names.values().cloned().collect::>())?; - let private = names - .iter() - .filter_map( - |(address, name)| match places.get(name).map(Vec::as_slice) { - Some([place]) => Some((*address, *place)), - _ => None, - }, - ) - .collect(); - let (mut row, mut report) = row_port::port_row( - &prior, - &named, - &NewBuild { - version, - pe: &pe, - symbols: &new, - private: &private, - }, - ); - row_port::seed_screen_origin(&mut row, &screen_origin, &pe, &mut report); - report.review.extend(row_port::hand_only_review(&prior)); - let text = |value: &dyn erased::Json| value.pretty(); - std::fs::write(row_path, text(&row)).map_err(|error| error.to_string())?; - std::fs::write(report_path, text(&report)).map_err(|error| error.to_string())?; + }; + // Each class table of the prior row is found again by its views class's name too. + let classes = row_port::prior_classes(prior, &prior_pe); + let bytes = read(module)?; + let pe = PeFile::parse(&bytes).ok_or("the new module is not a 64-bit PE image")?; + let (mut row, mut report) = + row_port::find_candidate(prior, &prints, Some(&prior_pe), &classes, version, &pe); + row_port::leave_out_screen_origin(&mut row, &mut report); + report.review.extend(row_port::hand_only_review(prior)); println!( - "{} {}: {} of {} addresses, {} for review", - report.browser, - report.version, - report.mapped, - report.addresses, - report.review.len() + "exception tables: {} function starts in the prior build, {} in the new one; {} class tables named", + prior_pe.function_starts().len(), + pe.function_starts().len(), + classes.len() ); - Ok(report.review.is_empty()) + Ok((row, report)) } -/// Makes a candidate row for a build with no published PDB from the prior row's fingerprints. The -/// prior PDB names each prior address, so a person sees which function a review line is about. -#[allow(clippy::too_many_arguments)] +/// Makes a candidate row for a new build. True when nothing needs a person's review. fn find( prior_path: &Path, - old_pdb: &Path, - old_module: &str, + prior_module: &Path, module: &Path, version: &str, - channel: &str, - download: &str, row_path: &Path, report_path: &Path, ) -> Result { let prior = json_file(prior_path)?; - let stored = prior["fingerprints"].as_object(); - let prior_bytes = match old_module { - "-" => None, - path => Some(read(Path::new(path))?), - }; - let prior_pe = match &prior_bytes { - Some(bytes) => { - Some(PeFile::parse(bytes).ok_or("the prior module is not a 64-bit PE image")?) - } - None => None, - }; - let prints = match (stored, &prior_pe) { - (Some(stored), None) => stored.clone(), - (None, None) => { - return Err("the prior row has no fingerprints, and no prior module was given to read them from".into()); - } - (_, Some(pe)) => { - // Fingerprints read from another build would find the wrong functions. - let start = prior["frame"]["hwnd_for_view"] - .as_u64() - .and_then(|address| u32::try_from(address).ok()); - let prologue = prior["frame"]["hwnd_for_view_prologue"] - .as_str() - .and_then(chrome_build::hex); - match (start, prologue) { - (Some(start), Some(prologue)) - if pe.read(start, prologue.len()) == Some(prologue.as_slice()) => {} - _ => return Err("the prior module is not the build of the prior row".into()), - } - match stored { - Some(stored) => stored.clone(), - None => { - let (prints, missing) = row_port::fingerprints_of(&prior, pe); - if !missing.is_empty() { - println!( - "the prior build gives no fingerprint for {}", - missing.join(", ") - ); - } - prints - } - } - } - }; - // A fingerprint taken at a function start counts only function starts of the new build, and - // what fingerprints leave open is looked for by reading the prior build next to the new one. - let prior_starts = prior_pe - .as_ref() - .map(PeFile::function_starts) - .unwrap_or_default(); - let mut old = Vec::new(); - symbols::publics(old_pdb, &mut |address, name| { - old.push((address, name.to_owned())) - })?; - old.sort(); - let mut targets = row_port::addresses_of(&prior["build"]); - for key in ["frame_view_vtable", "client_view_bounds", "hwnd_for_view"] { - targets.extend(prior["frame"][key].as_u64().map(|value| value as u32)); - } - let named = row_port::name_addresses(&targets, &old); - drop(old); - let bytes = read(module)?; - let pe = PeFile::parse(&bytes).ok_or("not a 64-bit PE image")?; - // A class table of a views class is also found again by the class name its symbol gives. - let classes = row_port::class_names(&named); - let (mut row, mut report) = - row_port::find_candidate(&prior, &prints, prior_pe.as_ref(), &classes, version, &pe); - // No symbols name `screen_origin` here: a prior row without it gives a row without it. - row_port::seed_screen_origin(&mut row, &[], &pe, &mut report); - report.review.extend(row_port::hand_only_review(&prior)); - row_port::name_review_lines(&mut report.review, &named); - let key = pe - .pdb_key() - .ok_or("the new module has no CodeView record")?; - row_port::mark_without_pdb(&mut row, &key, &report.prior_version, channel, download); + let (row, report) = make(&prior, prior_module, module, version)?; let text = |value: &dyn erased::Json| value.pretty(); std::fs::write(row_path, text(&row)).map_err(|error| error.to_string())?; std::fs::write(report_path, text(&report)).map_err(|error| error.to_string())?; println!( - "exception tables: {} function starts in the prior build, {} in the new one", - prior_starts.len(), - pe.function_starts().len() - ); - println!( - "{} {} without its PDB: {} of {} addresses found by fingerprint, {} settled by two methods, {} values read, {} carried, {} for review", + "{} {} from {}: {} of {} addresses found by fingerprint, {} settled by two methods, {} values read, {} carried, {} features off, {} for review", report.browser, report.version, + report.prior_version, report.fingerprinted.len(), report.addresses, report.settled.len(), report.derived.len(), report.carried.len(), + report.switched_off.len(), report.review.len() ); Ok(report.review.is_empty()) } -/// Compares a row made without its PDB with the row the PDB gives, and removes the mark when they -/// agree. -fn pdb_check(table_path: &Path, browser: &str, version: &str, row: &Path) -> Result { - let mut table = json_file(table_path)?; - let from_pdb = json_file(row)?; - let differences = row_port::check_against_pdb(&mut table, browser, version, &from_pdb)?; - for line in &differences { - println!("{line}"); +/// Makes the row of a build that already has a verified row, and scores it against that row. +/// True when no value it found is wrong. Nothing is written but the score. +fn score( + prior_path: &Path, + prior_module: &Path, + module: &Path, + verified_path: &Path, + out: &Path, +) -> Result { + let prior = json_file(prior_path)?; + let verified = json_file(verified_path)?; + let version = verified["frame"]["version"] + .as_str() + .ok_or("the verified row has no frame version")? + .to_owned(); + let (row, report) = make(&prior, prior_module, module, &version)?; + let score = row_port::score_row(&row, &report, &verified); + std::fs::write( + out, + serde_json::to_string_pretty(&score).map_err(|error| error.to_string())? + "\n", + ) + .map_err(|error| error.to_string())?; + println!( + "{} {} from {}:", + score.browser, score.version, score.prior_version + ); + for (feature, each) in &score.features { + let state = if each.off { "off" } else { "on" }; + println!( + " {feature}: {state}, {} equal, {} different, {} not held", + each.equal, + each.different.len(), + each.not_held.len() + ); + for line in &each.different { + println!(" wrong: {line}"); + } } - if !differences.is_empty() { - return Ok(false); + if !score.review.is_empty() { + println!( + " the row would stop for review: {}", + score.review.join("; ") + ); } - let text = serde_json::to_string_pretty(&table).map_err(|error| error.to_string())? + "\n"; - std::fs::write(table_path, text).map_err(|error| error.to_string())?; - println!("{browser} {version}: the row made from fingerprints agrees with its PDB"); - Ok(true) + Ok(score.wrong() == 0) } mod erased { @@ -417,18 +270,9 @@ fn pending( Ok(()) } -fn table_add(table_path: &Path, row: &Path, module: &Path) -> Result<(), String> { +fn table_add(table_path: &Path, row: &Path) -> Result<(), String> { let row = json_file(row)?; let mut table = json_file(table_path)?; - let bytes = read(module)?; - let pe = PeFile::parse(&bytes).ok_or("not a 64-bit PE image")?; - let key = pe.pdb_key().ok_or("the image has no CodeView record")?; - let name = format!( - "{} {}", - row["frame"]["browser"].as_str().unwrap_or_default(), - row["frame"]["version"].as_str().unwrap_or_default() - ); - table["symbols"][name] = key.into(); signed_table::upsert_row(&mut table, row)?; let text = serde_json::to_string_pretty(&table).map_err(|error| error.to_string())? + "\n"; std::fs::write(table_path, text).map_err(|error| error.to_string()) @@ -513,373 +357,6 @@ fn table_withdraw(published: &Path, signed: &Path, builds: &[&str]) -> Result<() Ok(()) } -fn pdb_key(module: &Path) -> Result<(), String> { - let bytes = read(module)?; - let pe = PeFile::parse(&bytes).ok_or("not a 64-bit PE image")?; - let key = pe.pdb_key().ok_or("the image has no CodeView record")?; - let pdb = format!( - "{}.pdb", - module - .file_name() - .and_then(|name| name.to_str()) - .ok_or("bad module name")? - ); - println!("{pdb}/{key}/{pdb}"); - Ok(()) -} - -fn extract( - module: &Path, - pdbs: &Path, - browser: &str, - version: &str, - table_path: &Path, - frame_class: &str, -) -> Result<(), String> { - let bytes = read(module)?; - let pe = PeFile::parse(&bytes).ok_or("not a 64-bit PE image")?; - let symbols = symbols::Session::open(module, pdbs)?; - // The function is declared on `BrowserFrameViewWin` whichever subclass builds the windows. - let client_view_bounds = symbols.address("BrowserFrameViewWin::GetBoundsForClientView")?; - let hwnd_for_view = symbols.address("views::HWNDForView")?; - // The class has several vtables because of multiple inheritance. Exactly one of them holds - // the client-view bounds function. - let (frame_view_vtable, client_view_slot) = symbols - .vtables(frame_class)? - .into_iter() - .find_map(|vtable| Some((vtable, pe.vtable_slot(vtable, client_view_bounds)?))) - .ok_or("no vtable of the frame class holds GetBoundsForClientView")?; - let prologue = pe - .read(hwnd_for_view, PROLOGUE) - .ok_or("views::HWNDForView is outside the image")?; - let build = ChromeBuild { - browser: browser.into(), - version: version.into(), - module: module - .file_name() - .and_then(|name| name.to_str()) - .ok_or("bad module name")? - .into(), - frame_view_vtable, - client_view_slot, - client_view_bounds, - hwnd_for_view, - hwnd_for_view_prologue: prologue.iter().map(|byte| format!("{byte:02x}")).collect(), - }; - let text = std::fs::read_to_string(table_path).unwrap_or_else(|_| "[]".into()); - let mut table = chrome_build::builds(&text); - chrome_build::upsert(&mut table, build.clone()); - let text = serde_json::to_string_pretty(&table).map_err(|error| error.to_string())? + "\n"; - std::fs::write(table_path, text) - .map_err(|error| format!("could not write {}: {error}", table_path.display()))?; - println!( - "{}", - serde_json::to_string(&build).map_err(|error| error.to_string())? - ); - Ok(()) -} - -#[cfg(windows)] -mod symbols { - use std::collections::{BTreeMap, HashMap}; - use std::os::windows::ffi::OsStrExt; - use std::path::Path; - - use windows_sys::Win32::System::Diagnostics::Debug::{ - SymCleanup, SymEnumSymbolsW, SymFromAddrW, SymFromNameW, SymInitializeW, SymLoadModuleExW, - SymSetOptions, SYMBOL_INFOW, SYMOPT_UNDNAME, - }; - use windows_sys::Win32::System::Threading::GetCurrentProcess; - - /// The base the module is loaded at for symbol lookups. Results are reported as RVAs. - const BASE: u64 = 0x1_8000_0000; - /// Accept a PDB whose age differs from the image, which Google's symbol server serves. - const SYMOPT_LOAD_ANYTHING: u32 = 0x40; - - fn wide(path: &Path) -> Vec { - path.as_os_str().encode_wide().chain(Some(0)).collect() - } - - /// Where a PDB opened on its own is placed: wide enough for every browser's module. - const IMAGE_SIZE: u32 = 0x2000_0000; - /// Reads only the linker's public names, which are decorated and stable between builds. - const SYMOPT_PUBLICS_ONLY: u32 = 0x4000; - - /// Visits every public symbol of a PDB, with its address in the module and its decorated name. - pub fn publics(pdb: &Path, visit: &mut dyn FnMut(u32, &str)) -> Result<(), String> { - unsafe extern "system" fn each( - info: *const SYMBOL_INFOW, - _size: u32, - context: *const core::ffi::c_void, - ) -> i32 { - let info = &*info; - let visit = &mut *(context as *mut &mut dyn FnMut(u32, &str)); - let name = std::slice::from_raw_parts(info.Name.as_ptr(), info.NameLen as usize); - if let Ok(address) = u32::try_from(info.Address - BASE) { - visit(address, &String::from_utf16_lossy(name)); - } - 1 - } - unsafe { - SymSetOptions(SYMOPT_LOAD_ANYTHING | SYMOPT_PUBLICS_ONLY); - if SymInitializeW(GetCurrentProcess(), std::ptr::null(), 0) == 0 { - return Err("DbgHelp could not start".into()); - } - let base = SymLoadModuleExW( - GetCurrentProcess(), - std::ptr::null_mut(), - wide(pdb).as_ptr(), - std::ptr::null(), - BASE, - IMAGE_SIZE, - std::ptr::null(), - 0, - ); - if base == 0 { - SymCleanup(GetCurrentProcess()); - return Err(format!("DbgHelp could not load {}", pdb.display())); - } - let mask: Vec = "*".encode_utf16().chain(Some(0)).collect(); - let mut visit = visit; - let done = SymEnumSymbolsW( - GetCurrentProcess(), - BASE, - mask.as_ptr(), - Some(each), - (&raw mut visit).cast(), - ); - SymCleanup(GetCurrentProcess()); - if done == 0 { - return Err(format!("DbgHelp could not list {}", pdb.display())); - } - } - Ok(()) - } - - fn load_all(pdb: &Path) -> Result<(), String> { - unsafe { - SymSetOptions(SYMOPT_LOAD_ANYTHING | SYMOPT_UNDNAME); - if SymInitializeW(GetCurrentProcess(), std::ptr::null(), 0) == 0 { - return Err("DbgHelp could not start".into()); - } - let base = SymLoadModuleExW( - GetCurrentProcess(), - std::ptr::null_mut(), - wide(pdb).as_ptr(), - std::ptr::null(), - BASE, - IMAGE_SIZE, - std::ptr::null(), - 0, - ); - if base == 0 { - SymCleanup(GetCurrentProcess()); - return Err(format!("DbgHelp could not load {}", pdb.display())); - } - } - Ok(()) - } - - /// The function that starts at each address, from the PDB's private symbols. - pub fn private_names(pdb: &Path, addresses: &[u32]) -> Result, String> { - load_all(pdb)?; - let mut names = BTreeMap::new(); - for address in addresses { - unsafe { - // SYMBOL_INFOW ends in the name, whose room the caller provides. - let mut storage = vec![0u64; (std::mem::size_of::() + 2048) / 8]; - let info = storage.as_mut_ptr().cast::(); - (*info).SizeOfStruct = std::mem::size_of::() as u32; - (*info).MaxNameLen = 1000; - let mut distance = 0u64; - if SymFromAddrW( - GetCurrentProcess(), - BASE + u64::from(*address), - &mut distance, - info, - ) != 0 - && distance == 0 - { - let name = - std::slice::from_raw_parts((*info).Name.as_ptr(), (*info).NameLen as usize); - names.insert( - *address, - String::from_utf16_lossy(name) - .trim_end_matches(['\0', ' ']) - .to_owned(), - ); - } - } - } - unsafe { SymCleanup(GetCurrentProcess()) }; - Ok(names) - } - - /// Where each function name lies, from the PDB's private symbols. - pub fn private_places( - pdb: &Path, - names: &[String], - ) -> Result>, String> { - unsafe extern "system" fn each( - info: *const SYMBOL_INFOW, - _size: u32, - context: *const core::ffi::c_void, - ) -> i32 { - let info = &*info; - let found = &mut *(context as *mut Vec<(String, u32)>); - let name = std::slice::from_raw_parts(info.Name.as_ptr(), info.NameLen as usize); - if let Ok(address) = u32::try_from(info.Address - BASE) { - found.push((String::from_utf16_lossy(name), address)); - } - 1 - } - load_all(pdb)?; - let mut places: HashMap> = HashMap::new(); - for name in names { - let mask: Vec = name.encode_utf16().chain(Some(0)).collect(); - let mut found: Vec<(String, u32)> = Vec::new(); - unsafe { - SymEnumSymbolsW( - GetCurrentProcess(), - BASE, - mask.as_ptr(), - Some(each), - (&raw mut found).cast(), - ); - } - let mut exact: Vec = found - .into_iter() - .filter(|(other, _)| other.trim_end_matches(['\0', ' ']) == name) - .map(|(_, place)| place) - .collect(); - exact.sort_unstable(); - exact.dedup(); - places.insert(name.clone(), exact); - } - unsafe { SymCleanup(GetCurrentProcess()) }; - Ok(places) - } - - pub struct Session; - - impl Session { - pub fn open(module: &Path, pdbs: &Path) -> Result { - unsafe { - SymSetOptions(SYMOPT_UNDNAME | SYMOPT_LOAD_ANYTHING); - if SymInitializeW(GetCurrentProcess(), wide(pdbs).as_ptr(), 0) == 0 { - return Err("DbgHelp could not start".into()); - } - let base = SymLoadModuleExW( - GetCurrentProcess(), - std::ptr::null_mut(), - wide(module).as_ptr(), - std::ptr::null(), - BASE, - 0, - std::ptr::null(), - 0, - ); - if base == 0 { - return Err("DbgHelp could not load the module's symbols".into()); - } - } - Ok(Self) - } - - pub fn address(&self, name: &str) -> Result { - let name_wide: Vec = name.encode_utf16().chain(Some(0)).collect(); - unsafe { - let mut info: SYMBOL_INFOW = std::mem::zeroed(); - info.SizeOfStruct = std::mem::size_of::() as u32; - if SymFromNameW(GetCurrentProcess(), name_wide.as_ptr(), &mut info) == 0 { - return Err(format!("the symbols have no {name}")); - } - u32::try_from(info.Address - BASE).map_err(|_| format!("{name} is out of range")) - } - } - - pub fn vtables(&self, class: &str) -> Result, String> { - unsafe extern "system" fn visit( - info: *const SYMBOL_INFOW, - _size: u32, - context: *const core::ffi::c_void, - ) -> i32 { - let info = &*info; - let found = &mut *(context as *mut Vec<(String, u64)>); - let name = std::slice::from_raw_parts(info.Name.as_ptr(), info.NameLen as usize); - found.push((String::from_utf16_lossy(name), info.Address)); - 1 - } - let mask: Vec = format!("{class}::*") - .encode_utf16() - .chain(Some(0)) - .collect(); - let mut found: Vec<(String, u64)> = Vec::new(); - unsafe { - SymEnumSymbolsW( - GetCurrentProcess(), - BASE, - mask.as_ptr(), - Some(visit), - &mut found as *mut _ as *const _, - ); - } - let vtables: Vec = found - .into_iter() - .filter(|(name, _)| { - name.trim_end_matches(['\0', ' ']) == format!("{class}::`vftable'") - }) - .filter_map(|(_, address)| u32::try_from(address - BASE).ok()) - .collect(); - if vtables.is_empty() { - return Err(format!("the symbols have no {class} vtable")); - } - Ok(vtables) - } - } - - impl Drop for Session { - fn drop(&mut self) { - unsafe { SymCleanup(GetCurrentProcess()) }; - } - } -} - -#[cfg(not(windows))] -mod symbols { - use std::path::Path; - - pub fn publics(_pdb: &Path, _visit: &mut dyn FnMut(u32, &str)) -> Result<(), String> { - Err("symbol extraction needs Windows DbgHelp".into()) - } - pub fn private_names( - _pdb: &Path, - _addresses: &[u32], - ) -> Result, String> { - Err("symbol extraction needs Windows DbgHelp".into()) - } - pub fn private_places( - _pdb: &Path, - _names: &[String], - ) -> Result>, String> { - Err("symbol extraction needs Windows DbgHelp".into()) - } - - pub struct Session; - - impl Session { - pub fn open(_module: &Path, _pdbs: &Path) -> Result { - Err("symbol extraction needs Windows DbgHelp".into()) - } - pub fn address(&self, _name: &str) -> Result { - unreachable!() - } - pub fn vtables(&self, _class: &str) -> Result, String> { - unreachable!() - } - } -} - /// Refuses a browser that did not pass every feature check, so it is never listed as supported. /// The row is the one being published: a check of a feature it turns off is not required. fn gate(results: &Path, browser: &str, row: Option<&Path>) -> Result<(), String> { diff --git a/rivet/platform/windows/src/chrome_build.rs b/rivet/platform/windows/src/chrome_build.rs index 7328df3..5805740 100644 --- a/rivet/platform/windows/src/chrome_build.rs +++ b/rivet/platform/windows/src/chrome_build.rs @@ -2,9 +2,8 @@ //! new ones. //! //! Each entry records where one build keeps the frame view's client-view bounds function. The -//! runtime only patches a loaded module that matches an entry exactly. The `chrome-builds` tool -//! fills entries from Google's public symbols, so a new browser release needs a data change, not -//! a code change. +//! runtime only patches a loaded module that matches an entry exactly, so a new browser release +//! needs a data change, not a code change. use serde::{Deserialize, Serialize}; @@ -61,13 +60,12 @@ pub fn hex(value: &str) -> Option> { .collect() } -/// A PE image read from a file, with enough structure to map RVAs and find its debug record. +/// A PE image read from a file, with enough structure to map RVAs and find its functions. pub struct PeFile<'a> { bytes: &'a [u8], image_base: u64, sections: Vec<(u32, u32, u32)>, code_sections: Vec<(u32, u32)>, - debug_directory: (u32, u32), exception_directory: (u32, u32), } @@ -103,11 +101,6 @@ impl<'a> PeFile<'a> { return None; } let image_base = u64_at(bytes, optional + 24)?; - // Data directory 6 is the debug directory. - let debug_directory = ( - u32_at(bytes, optional + 112 + 48)?, - u32_at(bytes, optional + 116 + 48)?, - ); // Data directory 3 is the exception table, which lists where functions start. let exception_directory = ( u32_at(bytes, optional + 112 + 24)?, @@ -142,7 +135,6 @@ impl<'a> PeFile<'a> { image_base, sections, code_sections, - debug_directory, exception_directory, }) } @@ -191,32 +183,6 @@ impl<'a> PeFile<'a> { u32::try_from(value.checked_sub(self.image_base)?).ok() } - /// The symbol-server key of the image's PDB: GUID and age from its CodeView record. - pub fn pdb_key(&self) -> Option { - let (directory, size) = self.debug_directory; - for index in 0..size / 28 { - let entry = self.offset(directory + index * 28)?; - // Type 2 is CodeView. - if u32_at(self.bytes, entry + 12)? != 2 { - continue; - } - let record = u32_at(self.bytes, entry + 24)? as usize; - if self.bytes.get(record..record + 4)? != b"RSDS" { - return None; - } - let guid = self.bytes.get(record + 4..record + 20)?; - let age = u32_at(self.bytes, record + 20)?; - let tail: String = guid[8..].iter().map(|byte| format!("{byte:02X}")).collect(); - return Some(format!( - "{:08X}{:04X}{:04X}{tail}{age:X}", - u32_at(guid, 0)?, - u16_at(guid, 4)?, - u16_at(guid, 6)?, - )); - } - None - } - /// The start of every function the exception table lists. A function that needs no unwind /// data, such as a leaf that touches no stack, is not listed. pub fn function_starts(&self) -> std::collections::BTreeSet { @@ -288,23 +254,6 @@ mod tests { bytes } - #[test] - fn nfr3_reads_the_symbol_server_key_of_an_image() { - let mut section = vec![0u8; 0x100]; - section[12..16].copy_from_slice(&2u32.to_le_bytes()); - section[24..28].copy_from_slice(&0x240u32.to_le_bytes()); - section[0x40..0x44].copy_from_slice(b"RSDS"); - let guid = [ - 0x51, 0x19, 0x13, 0x7b, 0xa2, 0x33, 0x0f, 0x9e, 0x4c, 0x4c, 0x44, 0x20, 0x50, 0x44, - 0x42, 0x2e, - ]; - section[0x44..0x54].copy_from_slice(&guid); - section[0x54..0x58].copy_from_slice(&1u32.to_le_bytes()); - let bytes = image(§ion); - let pe = PeFile::parse(&bytes).unwrap(); - assert_eq!(pe.pdb_key().unwrap(), "7B13195133A29E0F4C4C44205044422E1"); - } - #[test] fn nfr3_finds_a_function_in_a_vtable() { let mut section = vec![0u8; 0x100]; diff --git a/rivet/platform/windows/src/row_class.rs b/rivet/platform/windows/src/row_class.rs index 6a0112a..c696e40 100644 --- a/rivet/platform/windows/src/row_class.rs +++ b/rivet/platform/windows/src/row_class.rs @@ -366,13 +366,170 @@ pub fn class_tables(pe: &PeFile, name: &str) -> BTreeSet { found } -/// The class a public table symbol names, as `??_7EdgeBrowserFrameViewWin@@6BLayerDelegate@ui@@@` -/// names `EdgeBrowserFrameViewWin`. The class name its metadata holds has no namespace. -pub fn class_of_table_symbol(symbol: &str) -> Option<&str> { - let rest = symbol.strip_prefix("??_7")?; - let name = rest.split('@').next()?; - (!name.is_empty() && name.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')) - .then_some(name) +/// How many levels of callees below the code that fills the metadata are read for its name. +const CALLEE_DEPTH: usize = 2; +/// How many names may be tried for one table before it counts as too unclear to name. +const NAMES_MOST: usize = 16; +/// How long a class name may be. +const NAME_MOST: usize = 120; + +/// The zero-terminated text at `rva` when it reads as a C++ class name without its namespace. +fn identifier_at(pe: &PeFile, rva: u32) -> Option { + let bytes = [NAME_MOST + 1, 64, 32, 16, 8] + .into_iter() + .find_map(|length| pe.read(rva, length))?; + let end = bytes.iter().position(|byte| *byte == 0)?; + let name = &bytes[..end]; + let first = *name.first()?; + (name.len() >= 2 + && (first.is_ascii_alphabetic() || first == b'_') + && name + .iter() + .all(|byte| byte.is_ascii_alphanumeric() || *byte == b'_')) + .then(|| String::from_utf8_lossy(name).into_owned()) +} + +/// What a function's body loads with a RIP-relative `lea` outside the code, and the functions it +/// calls or jumps to directly. +fn body_of(pe: &PeFile, ranges: &BTreeMap, start: u32) -> (Vec, Vec) { + let end = ranges.get(&start).copied().unwrap_or(start); + let length = end.saturating_sub(start).min(BODY_MOST) as usize; + let Some(bytes) = pe.read(start, length) else { + return (vec![], vec![]); + }; + let (mut loads, mut callees) = (Vec::new(), Vec::new()); + let mut at = 0; + while at < bytes.len() { + let Some(instruction) = x64::decode_body(&bytes[at..]) else { + break; + }; + if instruction.length == 0 || at + instruction.length > bytes.len() { + break; + } + let raw = &bytes[at..at + instruction.length]; + let rva = start + at as u32; + if let Some(found) = (raw.len() == 7).then(|| rip_use(raw, rva)).flatten() { + if !found.word && !pe.is_code(found.target) { + loads.push(found.target); + } + } + if raw.len() == 5 && matches!(raw[0], 0xE8 | 0xE9) { + let rel = i32::from_le_bytes(raw[1..5].try_into().expect("five bytes")); + if let Ok(target) = u32::try_from(i64::from(rva) + 5 + i64::from(rel)) { + if ranges.contains_key(&target) { + callees.push(target); + } + } + } + at += instruction.length; + } + (loads, callees) +} + +/// The metadata statics a class table's objects reach: each constructor that writes the table at +/// the start of an object also writes other tables into it, and the second slot of the +/// `ui::metadata::MetaDataProvider` table among them returns the class's static, as `getters` reads. +fn statics_of_table(pe: &PeFile, ranges: &BTreeMap, table: u32) -> BTreeSet<(u32, bool)> { + let writers: BTreeSet = rip_uses(pe, &BTreeSet::from([table])) + .iter() + .filter(|each| !each.word) + .filter_map(|each| enclosing(ranges, each.at)) + .collect(); + let mut found = BTreeSet::new(); + for writer in writers { + let stores = stores_in(pe, writer, ranges[&writer]); + for first in stores + .iter() + .filter(|store| store.table == table && store.offset == 0) + { + for other in stores.iter().filter(|store| { + store.base == first.base && store.offset != 0 && store.table != table + }) { + let Some(getter) = pe.pointer_rva(other.table + 8).filter(|at| pe.is_code(*at)) + else { + continue; + }; + if let Some(each) = load_target(pe, getter).filter(|(at, _)| !pe.is_code(*at)) { + found.insert(each); + } + } + } + } + found +} + +/// The names the code that fills a metadata static loads, nearest first: the texts that the +/// functions whose result is the static, and their callees, load. Only those that read as a class +/// name are kept. +fn names_of_static( + pe: &PeFile, + ranges: &BTreeMap, + found: (u32, bool), +) -> Vec> { + let mut layer: BTreeSet = rip_uses(pe, &BTreeSet::from([found.0])) + .iter() + .filter_map(|each| enclosing(ranges, each.at)) + .filter(|function| load_target(pe, *function) == Some(found)) + .collect(); + let mut seen = layer.clone(); + let mut by_depth = Vec::new(); + for _ in 0..=CALLEE_DEPTH { + if layer.is_empty() || seen.len() > CALLERS_MOST { + break; + } + let mut names = Vec::new(); + let mut next = BTreeSet::new(); + for function in &layer { + let (loads, callees) = body_of(pe, ranges, *function); + names.extend(loads.into_iter().filter_map(|at| identifier_at(pe, at))); + next.extend(callees.into_iter().filter(|callee| !seen.contains(callee))); + } + by_depth.push(names); + seen.extend(&next); + layer = next; + } + by_depth +} + +/// The views class a class table belongs to, by the class name its `ui::metadata` holds, read back +/// from the build: the metadata static the table's objects reach, and the names the code that fills +/// it loads. A name counts only where `class_tables` gives exactly this table for it, so a name +/// that is not the class's own gives none. The nearest code that yields exactly one such name +/// names the table; none, or two, gives no name. +pub fn class_name_of_table(pe: &PeFile, table: u32) -> Option { + let ranges = pe.function_ranges(); + let mut by_depth: Vec> = Vec::new(); + for each in statics_of_table(pe, &ranges, table) { + for (depth, names) in names_of_static(pe, &ranges, each).into_iter().enumerate() { + if by_depth.len() <= depth { + by_depth.resize(depth + 1, Vec::new()); + } + by_depth[depth].extend(names); + } + } + let mut tried = BTreeSet::new(); + for names in by_depth { + let mut confirmed = BTreeSet::new(); + for name in names { + if tried.len() >= NAMES_MOST { + return None; + } + if tried.insert(name.clone()) && class_tables(pe, &name) == BTreeSet::from([table]) { + confirmed.insert(name); + } + } + match confirmed.len() { + 0 => continue, + 1 => return confirmed.into_iter().next(), + _ => return None, + } + } + None +} + +/// Whether `name` is the class name of `table` in `pe`: `class_tables` gives this table alone. +pub fn names_table(pe: &PeFile, name: &str, table: u32) -> bool { + class_tables(pe, name) == BTreeSet::from([table]) } #[cfg(test)] @@ -518,15 +675,33 @@ mod tests { } #[test] - fn nfr6_a_table_symbol_names_its_class() { + fn nfr6_a_class_name_is_read_from_the_prior_build_without_symbols() { + let bytes = class_module("EdgeBrowserFrameViewWin"); + let pe = PeFile::parse(&bytes).unwrap(); + // The table the constructor writes at the object's start is named by the text its + // metadata code loads, and the name gives that table back. assert_eq!( - class_of_table_symbol("??_7EdgeBrowserFrameViewWin@@6BLayerDelegate@ui@@@"), + class_name_of_table(&pe, 0x2400).as_deref(), Some("EdgeBrowserFrameViewWin") ); - assert_eq!( - class_of_table_symbol("??_7VerticalTabStripStateControllerImpl@tabs@@6B@"), - Some("VerticalTabStripStateControllerImpl") - ); - assert_eq!(class_of_table_symbol("?Layout@BrowserView@@UEAAXXZ"), None); + assert!(names_table(&pe, "EdgeBrowserFrameViewWin", 0x2400)); + // The provider table is written further into the object, so it names no class. + assert_eq!(class_name_of_table(&pe, 0x2200), None); + assert!(!names_table(&pe, "EdgeBrowserFrameViewWin", 0x2200)); + // A name the metadata code does not load does not name the table. + assert!(!names_table(&pe, "BrowserFrameViewWin", 0x2400)); + } + + #[test] + fn nfr6_a_class_name_is_a_plain_identifier() { + let mut code = class_module("BrowserView"); + let at = 0x400 + 0x1000 + 0x100; + assert_eq!(&code[at..at + 12], b"BrowserView\0"); + let pe = PeFile::parse(&code).unwrap(); + assert_eq!(identifier_at(&pe, 0x2100).as_deref(), Some("BrowserView")); + // A path or a qualified name is not a class name. + code[at + 7] = b':'; + let pe = PeFile::parse(&code).unwrap(); + assert_eq!(identifier_at(&pe, 0x2100), None); } } diff --git a/rivet/platform/windows/src/row_port.rs b/rivet/platform/windows/src/row_port.rs index c574c68..f656976 100644 --- a/rivet/platform/windows/src/row_port.rs +++ b/rivet/platform/windows/src/row_port.rs @@ -1,19 +1,20 @@ //! Makes the supported-build row of a new browser build from the row of a prior build. //! //! Chromium's source changes little between two builds of a milestone, but every address moves. -//! So each address of the prior row is named through the prior build's public symbols, the same -//! decorated names are looked up in the new build's symbols, and the instruction bytes that guard -//! each hook are read again from the new module. Values that are not addresses (member offsets, +//! Each address of the prior row is found again in the new module by its fingerprint, the first +//! instructions of the function without their displacements, or by a slot of a class table that +//! holds such a function. What a fingerprint does not pin down is settled by `row_resolve`, which +//! takes a place only where two independent methods agree. The instruction bytes that guard each +//! hook are then read again from the new module. Values that are not addresses (member offsets, //! table slots, object sizes) are read again from the new module where `row_recipe` has a recipe -//! for them, and a recipe that fails stops the row. The rest are carried over. They have no public -//! symbols, so a row that crosses a milestone lists each of them and is never published without a -//! person's review, and every row must still pass on the real browser. +//! for them, or from the instructions that use them. No browser's debug symbols are read: the +//! verified prior row, its fingerprints and the two modules are all the generator has. //! //! A row is plain data here, in the form the runtime writes: a `usize` is a `0x` string, and bytes -//! are bare hex. A value this code cannot explain stops the row: it goes to review rather than to -//! people's browsers, and the runtime's own prologue checks stand behind that. +//! are bare hex. A value this code cannot explain stops the row or turns its feature off: it never +//! reaches people's browsers, and the runtime's own prologue checks stand behind that. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use serde::Serialize; use serde_json::{json, Value}; @@ -25,22 +26,6 @@ use crate::row_resolve; const LOWEST_ADDRESS: u64 = 0x10_0000; /// The address a build records for what it does not have. const ABSENT: u64 = 0x7FFF_0000; -/// How far below an address its naming symbol may lie: a table slot or an instruction in a function. -const NEAREST_SYMBOL: u32 = 0x2000; - -/// One symbol: its address in the module and its decorated name. -pub type Symbol = (u32, String); - -/// Removes what the linker adds to a decorated name, which differs from build to build. -pub fn stable_name(name: &str) -> &str { - let name = name.trim_end_matches('\0'); - let name = match name.strip_prefix(".weak.") { - Some(rest) => rest.split(".default.").next().unwrap_or(rest), - None => name, - }; - name.split(".llvm.").next().unwrap_or(name) -} - /// What the generator found, for the person or job that decides what to do with the row. #[derive(Debug, Default, Serialize)] pub struct PortReport { @@ -53,10 +38,6 @@ pub struct PortReport { pub review: Vec, /// Addresses with several possible new places, settled by an independent check. pub settled: Vec, - /// Addresses inside a symbol rather than at its start, moved by the same distance. - pub interior: Vec, - /// Addresses found through private symbols, since no public one covers them. - pub private: Vec, pub bytes_changed: Vec, pub slots_moved: Vec, /// Values that are not addresses, read again from the new module by their recipes. @@ -65,8 +46,6 @@ pub struct PortReport { pub carried: Vec, /// Optional parts the new build could not supply. The runtime does without them. pub left_out: Vec, - /// The new build had no symbols, so every address was looked for by the prior row's fingerprints. - pub made_without_pdb: bool, /// Addresses a fingerprint found in exactly one place of the new build. pub fingerprinted: Vec, /// Values that are Rivet's settings rather than facts about the build, kept from the prior row. @@ -140,56 +119,6 @@ pub fn addresses_of(row: &Value) -> BTreeSet { found } -/// Names each address: all the names of the nearest symbol at or below it, which are many where -/// the linker folded identical functions, and how far above that symbol it lies. -pub fn name_addresses( - targets: &BTreeSet, - sorted_symbols: &[Symbol], -) -> BTreeMap)> { - let mut named = BTreeMap::new(); - for target in targets { - let end = sorted_symbols.partition_point(|(address, _)| address <= target); - let Some((nearest, _)) = end.checked_sub(1).map(|at| &sorted_symbols[at]) else { - continue; - }; - if target - nearest > NEAREST_SYMBOL { - continue; - } - let start = sorted_symbols.partition_point(|(address, _)| address < nearest); - let names = sorted_symbols[start..end] - .iter() - .map(|(_, name)| stable_name(name).to_owned()) - .collect(); - named.insert(*target, (target - nearest, names)); - } - named -} - -/// The names to keep from the new build's symbols. -pub fn wanted_names(named: &BTreeMap)>) -> HashSet { - named - .values() - .flat_map(|(_, names)| names.iter().cloned()) - .collect() -} - -/// Each place an address may have moved to, with how many of its names say so, most votes first. -fn candidates( - names: &[String], - distance: u32, - new: &HashMap>, -) -> Vec<(u32, usize)> { - let mut votes: BTreeMap = BTreeMap::new(); - for name in names { - for address in new.get(name).map_or(&[][..], Vec::as_slice) { - *votes.entry(address + distance).or_default() += 1; - } - } - let mut votes: Vec<_> = votes.into_iter().collect(); - votes.sort_by_key(|(address, count)| (std::cmp::Reverse(*count), *address)); - votes -} - /// A function's instructions, read as far as this needs them. pub mod x64 { /// One decoded instruction: its length, where its RIP-relative displacement or relative branch @@ -622,49 +551,6 @@ fn usize_at(row: &Value, path: &str) -> Option { read_address(at_path(row, path)) } -/// Everything the new build supplies. -pub struct NewBuild<'a> { - pub version: &'a str, - pub pe: &'a PeFile<'a>, - pub symbols: &'a HashMap>, - /// Where each address with no public name went, found through private symbols. - pub private: &'a BTreeMap, -} - -/// Settles an address the names disagree on. Folded functions share names across builds, so the -/// names alone can point at a function the new build no longer folds with the wanted one. The -/// instructions that row checks for it say which of the places is meant, and a table slot that -/// must hold it says the same. -fn settle(target: u32, choices: &[(u32, usize)], build: &Value, pe: &PeFile) -> Option { - let mut remaining: Vec = choices.iter().map(|(address, _)| *address).collect(); - let mut guards = vec![]; - collect_guards(build, &mut guards); - for (address, bytes) in guards { - if address != target { - continue; - } - let wanted = x64::shape(&bytes)?; - remaining.retain(|place| { - pe.read(*place, bytes.len()) - .and_then(x64::shape) - .is_some_and(|shape| shape == wanted) - }); - } - if remaining.len() == 1 { - return Some(remaining[0]); - } - // The linker folds functions with the same code under one name, and a new build can fold them - // into two groups. Short functions read to their end show that any of the places does the same. - let code: Vec<_> = remaining - .iter() - .map(|place| whole_function(pe, *place)) - .collect(); - let first = code.first()?.as_ref()?; - code.iter() - .all(|other| other.as_ref() == Some(first)) - .then(|| remaining[0]) -} - /// The bytes of a function short enough to read to its end: its first `ret` with no branch before. pub(crate) fn whole_function(pe: &PeFile, address: u32) -> Option> { let bytes = pe.read(address, 48)?; @@ -968,53 +854,12 @@ fn read_rounded_corners(build: &mut Value, pe: &PeFile, report: &mut PortReport) build["split"]["set_rounded_corners_parity"] = write_address(start + 13); } -/// Addresses read from instructions above, which no symbol names. +/// Addresses read from instructions above, which no fingerprint finds. const READ_FROM_CODE: &[&str] = &[ "split.set_rounded_corners_changed", "split.set_rounded_corners_parity", ]; -fn read_from_code_targets(build: &Value) -> Vec { - READ_FROM_CODE - .iter() - .filter_map(|path| usize_at(build, path)) - .map(|value| value as u32) - .collect() -} - -/// The code addresses a row holds that no public symbol starts at, which need private symbols. -pub fn unnamed_code( - prior: &Value, - named: &BTreeMap)>, - pe_of_new: &PeFile, -) -> Vec { - let read = read_from_code_targets(&prior["build"]); - let mut out = vec![]; - for (target, (distance, _)) in named { - // The prior module is gone by now. Code and data keep their sections from build to build. - if *distance != 0 && !read.contains(target) && pe_of_new.is_code(*target) { - out.push(*target); - } - } - out -} - -/// Makes the new row. `prior` is `{frame, build}`, as the runtime writes it. -pub fn port_row( - prior: &Value, - named: &BTreeMap)>, - new: &NewBuild, -) -> (Value, PortReport) { - let (row, mut report) = begin_row(prior, new.version, true); - let mut moved: BTreeMap = BTreeMap::new(); - let targets = addresses_of(&row); - report.addresses = targets.len(); - port_by_name(prior, &row, named, new, &targets, &mut moved, &mut report); - let (mut row, mut report) = finish_row(row, &moved, new.pe, report); - settle_carried(&mut row, &mut report); - (row, report) -} - /// Settles each value no recipe or instruction read from the new build. A value the runtime finds /// by reflection is left out of the row. A value the runtime does not read in this row is written /// as zero. A value of a feature the runtime can do without turns @@ -1112,10 +957,6 @@ fn settle_carried(row: &mut Value, report: &mut PortReport) { /// row; without it they are read from the new build later, each with its own review line. fn begin_row(prior: &Value, version: &str, carries: bool) -> (Value, PortReport) { let mut row = prior.clone(); - if let Some(object) = row.as_object_mut() { - // The mark tells how the prior row was made, not this one. - object.remove(WITHOUT_PDB); - } let prior_version = prior["frame"]["version"] .as_str() .unwrap_or_default() @@ -1218,73 +1059,6 @@ fn command_targets(prior: &Value, version: &str) -> BTreeMap { } } -/// Finds each address of a row through the public names of both builds. -fn port_by_name( - prior: &Value, - row: &Value, - named: &BTreeMap)>, - new: &NewBuild, - targets: &BTreeSet, - moved: &mut BTreeMap, - report: &mut PortReport, -) { - for target in targets { - let Some((distance, names)) = named.get(target) else { - continue; - }; - // No public name covers this function: the distance from the nearest one means nothing, - // since another build lays those functions out differently. - if *distance != 0 - && new.pe.is_code(*target) - && !read_from_code_targets(&prior["build"]).contains(target) - { - match new.private.get(target) { - Some(place) => { - moved.insert(*target, *place); - report.private.push(format!("{target:#x} -> {place:#x}")); - } - None => report.review.push(format!( - "{target:#x} lies in a function with no name in both builds" - )), - } - continue; - } - let choices = candidates(names, *distance, new.symbols); - let place = match choices.as_slice() { - [] => None, - [(only, _)] => Some(*only), - many => { - let settled = settle(*target, many, &row["build"], new.pe); - let place = settled - .or_else(|| by_slot(*target, many, &row["build"], new.pe, named, new.symbols)); - match place { - Some(place) => report - .settled - .push(format!("{target:#x} -> {place:#x} of {many:x?}")), - None => { - // Most folded names still fold together, so the largest group is the likely - // place, and a person looks at it. - report.review.push(format!( - "{target:#x} has several places: {many:x?}, first of {:?}", - &names[..names.len().min(3)] - )); - } - } - place.or(Some(many[0].0)) - } - }; - if let Some(place) = place { - if *distance != 0 { - report.interior.push(format!( - "{target:#x} = {}+{distance:#x} -> {place:#x}", - names[0] - )); - } - moved.insert(*target, place); - } - } -} - /// Writes the new places into a row and reads every guard and table slot back from the new module. fn finish_row( mut row: Value, @@ -1499,7 +1273,7 @@ pub(crate) fn find_table(pe: &PeFile, slot: u32, function: u32) -> Vec { } /// Makes the row of a build that has no row from the fingerprints of a verified row of the same -/// milestone, without symbols. Each address must match exactly one place, no two addresses may +/// milestone, from the new module alone. Each address must match exactly one place, no two addresses may /// share one, and every function start and table slot is then checked again in the new module the /// way a ported row is. Any doubt is an error with its reasons, and then the runtime changes /// nothing in the browser. @@ -1514,10 +1288,13 @@ pub fn find_row(prior: &Value, version: &str, pe: &PeFile) -> Result (Value, PortReport) { let (mut row, mut report) = begin_row(prior, version, prior_pe.is_none()); - report.made_without_pdb = true; let prior_starts = prior_pe.map(PeFile::function_starts).unwrap_or_default(); let targets = fingerprint_targets(prior); report.addresses = targets.len(); @@ -1691,8 +1467,8 @@ pub fn find_candidate( .iter() .map(|(target, (_, want))| (*target, want.clone())) .collect(); - // The tables the prior build's public symbols name after a class are the tables of - // views classes the slot shift compares. + // The tables a class name was confirmed for in the prior build are the tables of views + // classes the slot shift compares. let views: BTreeSet = classes.keys().copied().collect(); let resolution = row_resolve::resolve(prior_pe, pe, &moved, &asked, &table_targets, &views); @@ -1726,7 +1502,9 @@ pub fn find_candidate( if let Some(object) = row.as_object_mut() { // They describe the prior build, so they would mislead in this row. object.remove("fingerprints"); + object.remove(CLASSES); } + record_classes(prior, &mut row, classes, &moved); if let Some(prior_pe) = prior_pe { read_carried(prior, &mut row, &moved, &followed, prior_pe, pe, &mut report); } @@ -1914,170 +1692,82 @@ fn read_carried( } } -/// The key of a row made without its build's symbols. It holds what the row job needs to make the -/// row again from the symbols once they are published, and to compare the two. The runtime reads -/// only `frame` and `build`, so the mark changes nothing in a browser. -pub const WITHOUT_PDB: &str = "made_without_pdb"; +/// The key of a row that names the views class of each of its class tables, by the path of the +/// value that holds the table, as `{"build.layout.frame_vtable": "BrowserFrameViewWin"}`. The +/// runtime reads only `frame` and `build`, so the names change nothing in a browser. +pub const CLASSES: &str = "classes"; -/// Marks a row made from fingerprints with the symbol key of its module, the prior row it was made -/// from, and how the row job fetches the build again. -pub fn mark_without_pdb( - row: &mut Value, - pdb_key: &str, - prior_version: &str, - channel: &str, - download: &str, -) { - row[WITHOUT_PDB] = json!({ - "pdb_key": pdb_key, - "prior_version": prior_version, - "channel": channel, - "download": download, - }); -} - -/// Where a row made from fingerprints differs from the row the build's own symbols give. A value -/// the fingerprint row left out is not a difference, since the runtime does without it. -pub fn differences_from_pdb_row(found: &Value, from_pdb: &Value) -> Vec { - fn walk(path: &str, found: &Value, pdb: &Value, out: &mut Vec) { - match (found, pdb) { - (Value::Null, _) => {} - (Value::Object(left), Value::Object(right)) => { - for (key, value) in left { - let other = right.get(key).unwrap_or(&Value::Null); - walk(&format!("{path}.{key}"), value, other, out); - } - } - (Value::Array(left), Value::Array(right)) if left.len() == right.len() => { - for (index, (a, b)) in left.iter().zip(right).enumerate() { - walk(&format!("{path}[{index}]"), a, b, out); - } - } - _ if found != pdb => { - out.push(format!( - "{path}: {found} from fingerprints, {pdb} from the PDB" - )); - } - _ => {} +/// The path of every address a row holds, the frame entry's included, as `build.layout.frame_vtable` +/// or `frame.frame_view_vtable`. +fn address_paths(row: &Value) -> Vec<(String, u32)> { + let mut out = Vec::new(); + let mut note = |path: &str, value: &mut Value| { + if READ_FROM_CODE.iter().any(|field| path.ends_with(field)) { + return; } - } - let mut out = vec![]; - for part in ["frame", "build"] { - walk(part, &found[part], &from_pdb[part], &mut out); - } - out -} - -/// Compares the marked row of `browser` `version` in `table` with the row made from the build's -/// symbols. When they agree the mark is removed, since the symbols now vouch for the row. The -/// differences are returned, and the table is left as it was when there are any. -pub fn check_against_pdb( - table: &mut Value, - browser: &str, - version: &str, - from_pdb: &Value, -) -> Result, String> { - let row = table["rows"] - .as_array_mut() - .ok_or("the table has no rows")? - .iter_mut() - .find(|row| row["frame"]["browser"] == browser && row["frame"]["version"] == version) - .ok_or_else(|| format!("the table has no row for {browser} {version}"))?; - if row.get(WITHOUT_PDB).is_none() { - return Err(format!( - "the row of {browser} {version} was not made without its PDB" - )); - } - let differences = differences_from_pdb_row(row, from_pdb); - if differences.is_empty() { - if let Some(object) = row.as_object_mut() { - object.remove(WITHOUT_PDB); + if let Some(address) = read_address(value).filter(|value| is_address(*value)) { + out.push((path.to_owned(), address as u32)); } - } - Ok(differences) + }; + let mut build = row["build"].clone(); + walk_addresses("build", &mut build, &mut note); + let mut frame = frame_addresses(&row["frame"]); + walk_addresses("frame", &mut frame, &mut note); + out } -/// The views class each class table of the prior row is named after by the prior build's public -/// symbols, for the tables found again by their class name. A table whose symbols name two classes -/// is left out. -pub fn class_names(named: &BTreeMap)>) -> BTreeMap { +/// The views class of each class table of the prior row, from the build alone. A name the prior +/// row records for the table is taken when it gives exactly that table in the prior module, and +/// a table with no such name is named by the class name its metadata holds, read back from the +/// prior module by `row_class`. Either way the name is the one the class's `BEGIN_METADATA` gives +/// in Chromium's source, and a table no name gives alone has none. +pub fn prior_classes(prior: &Value, prior_pe: &PeFile) -> BTreeMap { + let recorded = prior[CLASSES].as_object(); let mut out = BTreeMap::new(); - for (target, (distance, names)) in named { - if *distance != 0 { + let mut tried = BTreeSet::new(); + for (path, address) in address_paths(prior) { + if prior_pe.is_code(address) || out.contains_key(&address) { continue; } - let classes: BTreeSet<&str> = names - .iter() - .filter_map(|name| crate::row_class::class_of_table_symbol(stable_name(name))) - .collect(); - if let [only] = classes.into_iter().collect::>()[..] { - out.insert(*target, only.to_owned()); - } - } - out -} - -/// Adds to each review line that starts with a prior address the public name that address lies -/// in, from the prior build's symbols, so a person sees which function was not found. -pub fn name_review_lines(review: &mut [String], named: &BTreeMap)>) { - for line in review { - let Some(digits) = line.strip_prefix("0x") else { - continue; - }; - let end = digits - .find(|c: char| !c.is_ascii_hexdigit()) - .unwrap_or(digits.len()); - let Ok(address) = u32::from_str_radix(&digits[..end], 16) else { - continue; - }; - if let Some((distance, names)) = named.get(&address) { - if let Some(name) = names.first() { - line.push_str(&format!(" ({name}+{distance:#x} in the prior build)")); + let name = recorded + .and_then(|names| names.get(&path)) + .and_then(Value::as_str) + .filter(|name| crate::row_class::names_table(prior_pe, name, address)) + .map(str::to_owned); + let name = match name { + Some(name) => Some(name), + None if tried.insert(address) => { + crate::row_class::class_name_of_table(prior_pe, address) } + None => None, + }; + if let Some(name) = name { + out.insert(address, name); } } + out } -/// A folded function that fills a known table slot is the one in that slot of the new table. -fn by_slot( - target: u32, - choices: &[(u32, usize)], - build: &Value, - pe: &PeFile, - named: &BTreeMap)>, - symbols: &HashMap>, -) -> Option { - for (vtable_path, slot_path, function_path) in VTABLE_SLOTS { - if usize_at(build, function_path) != Some(u64::from(target)) { - continue; - } - let vtable = usize_at(build, vtable_path)? as u32; - let slot = usize_at(build, slot_path)? as u32; - let (distance, names) = named.get(&vtable)?; - let [(new_vtable, _)] = candidates(names, *distance, symbols)[..] else { - continue; - }; - let held = pe.pointer_rva(new_vtable + slot * 8)?; - if choices.iter().any(|(place, _)| *place == held) { - return Some(held); +/// Writes into the new row the class name of each class table it placed, by the path of each value +/// that holds the table. +fn record_classes( + prior: &Value, + row: &mut Value, + classes: &BTreeMap, + moved: &BTreeMap, +) { + let mut names = serde_json::Map::new(); + for (path, address) in address_paths(prior) { + if let Some(name) = classes + .get(&address) + .filter(|_| moved.contains_key(&address)) + { + names.insert(path, Value::String(name.clone())); } } - // The classes of hidden buttons keep the function of one known slot. - let slot = usize_at(build, "toolbar.theme_slot")? as u32; - for pair in build["toolbar"]["hidden"].as_array()? { - if read_address(&pair[1]) != Some(u64::from(target)) { - continue; - } - let (distance, names) = named.get(&(read_address(&pair[0])? as u32))?; - let [(new_vtable, _)] = candidates(names, *distance, symbols)[..] else { - continue; - }; - let held = pe.pointer_rva(new_vtable + slot * 8)?; - if choices.iter().any(|(place, _)| *place == held) { - return Some(held); - } + if !names.is_empty() { + row[CLASSES] = Value::Object(names); } - None } /// The frame entry keeps plain numbers. The walk over a row reads `0x` strings. @@ -2148,59 +1838,136 @@ fn check_frame_slot(frame: &mut Value, pe: &PeFile, report: &mut PortReport) { } } -/// The start of the public name of `views::View::ConvertPointToScreen(const View*, gfx::Point*)`: -/// a static function returning nothing. The other overload returns the point and is named `SA?AV`. -pub const SCREEN_ORIGIN_NAME: &str = "?ConvertPointToScreen@View@views@@SAX"; +/// Leaves `screen_origin`, `[address, first bytes]` of `views::View::ConvertPointToScreen`, out of a +/// row whose prior row had none. A prior row that had it was found again with every other address. +/// The runtime then keeps placing bubbles from the window's monitor, so the row is not stopped. +pub fn leave_out_screen_origin(row: &mut Value, report: &mut PortReport) { + if row["build"]["screen_origin"].is_array() { + return; + } + row["build"]["screen_origin"] = Value::Null; + report + .left_out + .push("build.screen_origin: the prior row has none".to_owned()); +} -/// How many bytes of that function's start the runtime compares before it ever calls it. -const SCREEN_ORIGIN_BYTES: usize = 12; +/// The group a value of a row is scored in: its feature, or the window, whose parts cannot be off. +const WINDOW: &str = "window"; + +/// How one feature of a row made without debug symbols compares with the verified row of the same +/// build, value by value. +#[derive(Debug, Default, Serialize, PartialEq, Eq)] +pub struct FeatureScore { + /// Values the made row holds exactly as the verified row does. + pub equal: usize, + /// Values the made row holds differently, with both. Any is a wrong value. + pub different: Vec, + /// The made row turns the feature off, so none of its values is used. + pub off: bool, + /// Values the made row does not hold or does not use: the runtime finds them by reflection, + /// does not read them in this build, or the prior row did not have them. + pub not_held: Vec, +} -/// Whether a public name is the one `screen_origin` is found by. -pub fn is_screen_origin_name(name: &str) -> bool { - stable_name(name).starts_with(SCREEN_ORIGIN_NAME) +/// The score of a made row against its answer key: each feature's comparison, and the review lines +/// that stop the row. +#[derive(Debug, Default, Serialize)] +pub struct RowScore { + pub browser: String, + pub version: String, + pub prior_version: String, + pub features: BTreeMap, + pub review: Vec, } -/// The first whole instructions of a function, at least `SCREEN_ORIGIN_BYTES` long. -fn function_start(pe: &PeFile, address: u32) -> Option> { - if !pe.is_code(address) { - return None; +impl RowScore { + /// How many values were found but differ from the verified row. Must be zero. + pub fn wrong(&self) -> usize { + self.features + .values() + .map(|score| score.different.len()) + .sum() } - let window = pe.read(address, 32)?; - let (ends, _) = x64::boundaries(window, SCREEN_ORIGIN_BYTES); - let end = *ends.iter().find(|end| **end >= SCREEN_ORIGIN_BYTES)?; - Some(window[..end].to_vec()) } -/// Adds `screen_origin`, `[address, first bytes]` of `views::View::ConvertPointToScreen`, to a row -/// whose prior row had none, by the function's public name in the new build. A prior row that had -/// it was ported with every other address. A build with no single place for the name goes without -/// it, and the runtime keeps placing bubbles from the window's monitor, so the row is not stopped. -pub fn seed_screen_origin(row: &mut Value, places: &[u32], pe: &PeFile, report: &mut PortReport) { - if row["build"]["screen_origin"].is_array() { - return; +/// Every value a row part holds that is not a list or object, by its path. +fn leaves(path: &str, value: &Value, out: &mut BTreeMap) { + match value { + Value::Object(object) => { + for (key, item) in object { + let next = if path.is_empty() { + key.clone() + } else { + format!("{path}.{key}") + }; + leaves(&next, item, out); + } + } + Value::Array(items) if !items.is_empty() => { + for (index, item) in items.iter().enumerate() { + leaves(&format!("{path}[{index}]"), item, out); + } + } + _ => { + out.insert(path.to_owned(), value.clone()); + } } - let mut places = places.to_vec(); - places.sort_unstable(); - places.dedup(); - let found = match places.as_slice() { - [only] => function_start(pe, *only).map(|bytes| (*only, bytes)), - _ => None, +} + +/// Compares a row made from a prior row with the row people verified for the same build. A value of +/// a feature the made row turns off is not compared, since the runtime does not use it. A value the +/// made row leaves out, or writes as zero because this build does not read it, is listed as not +/// held. Every other value must be equal, and one that differs is listed with both values. +pub fn score_row(made: &Value, report: &PortReport, verified: &Value) -> RowScore { + let off: BTreeSet<&str> = made[crate::row_features::SWITCHED_OFF] + .as_array() + .map_or(&[][..], Vec::as_slice) + .iter() + .filter_map(|entry| entry["feature"].as_str()) + .collect(); + let unused: BTreeSet<&str> = report + .unused + .iter() + .chain(&report.reflected) + .filter_map(|line| line.split(": ").next()) + .collect(); + let mut score = RowScore { + browser: report.browser.clone(), + version: report.version.clone(), + prior_version: report.prior_version.clone(), + review: report.review.clone(), + ..RowScore::default() }; - match found { - Some((address, bytes)) => { - row["build"]["screen_origin"] = - json!([write_address(u64::from(address)), hex_text(&bytes)]); - report.derived.push(format!( - "build.screen_origin: {SCREEN_ORIGIN_NAME} at {address:#x}" - )); - } - None => { - row["build"]["screen_origin"] = Value::Null; - report.left_out.push(format!( - "build.screen_origin: no single readable function named {SCREEN_ORIGIN_NAME}: {places:x?}" - )); + for part in ["frame", "build"] { + let (mut want, mut have) = (BTreeMap::new(), BTreeMap::new()); + leaves("", &verified[part], &mut want); + leaves("", &made[part], &mut have); + for (path, value) in want { + let feature = match part { + "build" => { + crate::row_features::feature_of(&path).map_or(WINDOW, |feature| feature.name) + } + _ => WINDOW, + }; + let entry = score.features.entry(feature.to_owned()).or_default(); + if off.contains(feature) { + entry.off = true; + continue; + } + let shown = format!("{part}.{path}"); + match have.get(&path) { + Some(found) if *found == value => entry.equal += 1, + None | Some(Value::Null) => entry.not_held.push(shown), + Some(_) if unused.iter().any(|unused| path.starts_with(unused)) => { + entry.not_held.push(shown) + } + Some(found) => entry + .different + .push(format!("{shown}: made {found}, verified {value}")), + } } } + score } /// The review items for the parts of the prior row that only a person writes today. A row made @@ -2276,58 +2043,33 @@ mod tests { }) } - fn symbols_old() -> Vec { - [ - (0x200000, "vt"), - (0x200100, "bounds"), - (0x200200, "hwnd"), - (0x200300, "paint"), - ] - .map(|(address, name)| (address, name.to_owned())) - .to_vec() - } - - fn places(rows: &[(&str, &[u32])]) -> HashMap> { - rows.iter() - .map(|(name, list)| ((*name).to_owned(), list.to_vec())) - .collect() - } - fn pointer(rva: u32) -> [u8; 8] { (IMAGE_BASE + u64::from(rva)).to_le_bytes() } + /// Places the prior row's addresses where `moved` says, as the fingerprints and `row_resolve` + /// would, and settles the rest of the row the way every new row is settled. fn port( prior: &Value, - new: &[(&str, &[u32])], + moved: &[(u32, u32)], code: &[(u32, &[u8])], version: &str, ) -> (Value, PortReport) { - let mut targets = addresses_of(&prior["build"]); - for key in ["frame_view_vtable", "client_view_bounds", "hwnd_for_view"] { - targets.insert(prior["frame"][key].as_u64().unwrap() as u32); - } - let named = name_addresses(&targets, &symbols_old()); let bytes = module(code); let pe = PeFile::parse(&bytes).unwrap(); - let symbols = places(new); - port_row( - prior, - &named, - &NewBuild { - version, - pe: &pe, - symbols: &symbols, - private: &BTreeMap::new(), - }, - ) - } - - const NEW_SYMBOLS: &[(&str, &[u32])] = &[ - ("vt", &[0x101000]), - ("bounds", &[0x101100]), - ("hwnd", &[0x101200]), - ("paint", &[0x101300]), + let (row, mut report) = begin_row(prior, version, true); + report.addresses = addresses_of(&row["build"]).len(); + let moved: BTreeMap = moved.iter().copied().collect(); + let (mut row, mut report) = finish_row(row, &moved, &pe, report); + settle_carried(&mut row, &mut report); + (row, report) + } + + const NEW_PLACES: &[(u32, u32)] = &[ + (0x200000, 0x101000), + (0x200100, 0x101100), + (0x200200, 0x101200), + (0x200300, 0x101300), ]; fn new_code() -> Vec<(u32, Vec)> { @@ -2345,47 +2087,9 @@ mod tests { } #[test] - fn nfr3_screen_origin_is_found_by_its_public_name_in_the_new_build() { - // push rsi; push rdi; sub rsp, 0x28; mov rsi, rdx; mov rdi, rcx; call. - let start = hex("56574883ec284889d64889cfe800000000").unwrap(); - let bytes = module(&[(0x101400, start.as_slice())]); - let pe = PeFile::parse(&bytes).unwrap(); - let mut row = prior(); - let mut report = PortReport::default(); - seed_screen_origin(&mut row, &[0x101400, 0x101400], &pe, &mut report); - assert_eq!( - row["build"]["screen_origin"], - json!(["0x101400", "56574883ec284889d64889cf"]) - ); - assert!(report.review.is_empty() && report.left_out.is_empty()); - // A row that already has it keeps the place the port gave it. - seed_screen_origin(&mut row, &[0x101500], &pe, &mut report); - assert_eq!(row["build"]["screen_origin"][0], "0x101400"); - // No name, or two places for it, leaves the part out without stopping the row. - for places in [&[][..], &[0x101400, 0x101500][..]] { - let mut row = prior(); - let mut report = PortReport::default(); - seed_screen_origin(&mut row, places, &pe, &mut report); - assert!(row["build"]["screen_origin"].is_null()); - assert_eq!(report.left_out.len(), 1); - assert!(report.review.is_empty()); - } - // Only the in-place overload, which returns nothing, is the one. - assert!(is_screen_origin_name( - "?ConvertPointToScreen@View@views@@SAXPEBV12@PEAVPoint@gfx@@@Z" - )); - assert!(!is_screen_origin_name( - "?ConvertPointToScreen@View@views@@SA?AVPoint@gfx@@PEBV12@AEBV34@@Z" - )); - assert!(!is_screen_origin_name( - "?ConvertRectToScreen@View@views@@SAXPEBV12@PEAVRect@gfx@@@Z" - )); - } - - #[test] - fn nfr3_a_new_row_takes_addresses_by_name_and_bytes_from_the_new_module() { + fn nfr3_a_new_row_takes_the_found_places_and_bytes_from_the_new_module() { let code = new_code(); - let (row, report) = port(&prior(), NEW_SYMBOLS, &code_refs(&code), "1.0.0.2"); + let (row, report) = port(&prior(), NEW_PLACES, &code_refs(&code), "1.0.0.2"); assert_eq!(report.review, Vec::::new()); assert_eq!(row["build"]["version"], json!([1, 0, 0, 2])); assert_eq!(row["frame"]["version"], "1.0.0.2"); @@ -2402,14 +2106,14 @@ mod tests { } #[test] - fn nfr3_a_name_the_new_build_lacks_stops_the_row() { + fn nfr3_an_address_with_no_new_place_stops_the_row() { let code = new_code(); - let symbols: Vec<_> = NEW_SYMBOLS + let places: Vec<_> = NEW_PLACES .iter() - .filter(|(name, _)| *name != "paint") + .filter(|(prior, _)| *prior != 0x200300) .copied() .collect(); - let (_, report) = port(&prior(), &symbols, &code_refs(&code), "1.0.0.2"); + let (_, report) = port(&prior(), &places, &code_refs(&code), "1.0.0.2"); assert!( report .review @@ -2428,7 +2132,7 @@ mod tests { // A wrong prior value shows that the new row does not copy it. prior["build"]["toolbar"]["dev"] = json!({"inspect": 1, "console": 2, "colours": []}); let code = new_code(); - let (row, report) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "158.0.8127.36"); + let (row, report) = port(&prior, NEW_PLACES, &code_refs(&code), "158.0.8127.36"); assert_eq!(row["build"]["toolbar"]["dev"]["inspect"], 40023); assert_eq!(row["build"]["toolbar"]["dev"]["console"], 40005); assert!( @@ -2444,11 +2148,11 @@ mod tests { .expect("the other values still cross the milestone"); assert!(!milestone.contains("toolbar.dev.console"), "{milestone}"); // A milestone whose header was not read offers no command rather than the prior one. - let (row, _) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "199.0.1.1"); + let (row, _) = port(&prior, NEW_PLACES, &code_refs(&code), "199.0.1.1"); assert_eq!(row["build"]["toolbar"]["dev"]["inspect"], 0); // A row that never offered the developer bar keeps offering none. prior["build"]["toolbar"]["dev"] = json!({"inspect": 0, "console": 0, "colours": []}); - let (row, _) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "158.0.8127.36"); + let (row, _) = port(&prior, NEW_PLACES, &code_refs(&code), "158.0.8127.36"); assert_eq!(row["build"]["toolbar"]["dev"]["console"], 0); } @@ -2473,7 +2177,7 @@ mod tests { #[test] fn nfr6_a_feature_with_an_unread_value_is_off_for_that_build() { let code = new_code(); - let (row, report) = port(&prior_with_features(), NEW_SYMBOLS, &code_refs(&code), "2.0.0.1"); + let (row, report) = port(&prior_with_features(), NEW_PLACES, &code_refs(&code), "2.0.0.1"); // Nothing holds the rest of the row back. assert_eq!(report.review, Vec::::new()); assert!(report.carried.is_empty(), "{:?}", report.carried); @@ -2508,7 +2212,7 @@ mod tests { json!({"host_view": "0x38", "page_area": "0x0", "contents_container": "0xd8"}); prior["build"]["toolbar"]["get_widget_slot"] = json!("0xa"); let code = new_code(); - let (row, report) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "2.0.0.1"); + let (row, report) = port(&prior, NEW_PLACES, &code_refs(&code), "2.0.0.1"); assert_eq!(report.review, Vec::::new()); assert!(report.carried.is_empty(), "{:?}", report.carried); assert_eq!(report.reflected.len(), 5, "{:?}", report.reflected); @@ -2528,7 +2232,7 @@ mod tests { let mut prior = prior_with_features(); prior["build"]["layout"] = json!({"view_bounds_offset": "0x1c8"}); let code = new_code(); - let (row, report) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "2.0.0.1"); + let (row, report) = port(&prior, NEW_PLACES, &code_refs(&code), "2.0.0.1"); assert_eq!(report.carried, ["layout.view_bounds_offset"]); assert!( report.review.iter().any(|line| line.contains("milestone") @@ -2543,7 +2247,7 @@ mod tests { #[test] fn nfr6_a_row_within_its_milestone_turns_nothing_off() { let code = new_code(); - let (row, report) = port(&prior_with_features(), NEW_SYMBOLS, &code_refs(&code), "1.0.0.2"); + let (row, report) = port(&prior_with_features(), NEW_PLACES, &code_refs(&code), "1.0.0.2"); assert_eq!(report.review, Vec::::new()); assert!(report.switched_off.is_empty()); assert_eq!(row["build"]["split"]["supported"], true); @@ -2553,7 +2257,7 @@ mod tests { #[test] fn nfr3_a_row_that_crosses_a_milestone_stops_for_review() { let code = new_code(); - let (_, report) = port(&prior(), NEW_SYMBOLS, &code_refs(&code), "2.0.0.1"); + let (_, report) = port(&prior(), NEW_PLACES, &code_refs(&code), "2.0.0.1"); assert!( report.review.iter().any(|line| line.contains("milestone")), "{:?}", @@ -2567,7 +2271,7 @@ mod tests { .into_iter() .filter(|(rva, _)| *rva != 0x101008) .collect(); - let (_, report) = port(&prior(), NEW_SYMBOLS, &code_refs(&code), "1.0.0.2"); + let (_, report) = port(&prior(), NEW_PLACES, &code_refs(&code), "1.0.0.2"); assert!( report .review @@ -2578,55 +2282,6 @@ mod tests { ); } - #[test] - fn nfr3_folded_names_that_stay_apart_are_settled_by_the_guard_or_stop_the_row() { - let mut code = new_code(); - code.push((0x101400, hex("565755534883ec584889ce48").unwrap())); - let two: &[u32] = &[0x101300, 0x101400]; - let symbols: Vec<_> = NEW_SYMBOLS - .iter() - .map(|(name, list)| { - if *name == "paint" { - (*name, two) - } else { - (*name, *list) - } - }) - .collect(); - let (row, report) = port(&prior(), &symbols, &code_refs(&code), "1.0.0.2"); - assert_eq!(row["build"]["toolbar"]["paint"], "0x101300"); - assert_eq!(report.settled.len(), 1); - assert!(report.review.is_empty(), "{:?}", report.review); - // Two places with the same start cannot be told apart. - let mut same = new_code(); - same.push((0x101400, hex("415741564155415456575553").unwrap())); - let (_, report) = port(&prior(), &symbols, &code_refs(&same), "1.0.0.2"); - assert!( - report - .review - .iter() - .any(|line| line.contains("several places")), - "{:?}", - report.review - ); - } - - #[test] - fn nfr3_a_function_with_no_public_name_is_not_moved_by_distance() { - let mut prior = prior(); - prior["build"]["toolbar"]["inside"] = json!("0x200340"); - let code = new_code(); - let (_, report) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "1.0.0.2"); - assert!( - report - .review - .iter() - .any(|line| line.contains("no name in both")), - "{:?}", - report.review - ); - } - #[test] fn nfr3_a_guard_a_hook_copies_stops_on_an_instruction_in_the_new_build() { let mut prior = prior(); @@ -2634,7 +2289,7 @@ mod tests { prior["build"]["toolbar"]["paint_prologue"] = json!("41574156565755534881ecb8000000"); let mut code = new_code(); code[2].1 = hex("4156565755534881eca00000004889ce0000").unwrap(); - let (row, report) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "1.0.0.2"); + let (row, report) = port(&prior, NEW_PLACES, &code_refs(&code), "1.0.0.2"); assert_eq!( row["build"]["toolbar"]["paint_prologue"], "4156565755534881eca0000000" @@ -2642,39 +2297,12 @@ mod tests { assert!(report.review.is_empty(), "{:?}", report.review); } - #[test] - fn nfr3_stable_name_drops_what_the_linker_adds() { - assert_eq!(stable_name("??1GURL@@QEAA@XZ\0"), "??1GURL@@QEAA@XZ"); - assert_eq!(stable_name("?F@@YAXXZ.llvm.123456"), "?F@@YAXXZ"); - assert_eq!( - stable_name(".weak.??_7BrowserView@@6B@.default.?RejectPromise@@"), - "??_7BrowserView@@6B@" - ); - } - #[test] fn nfr3_addresses_are_told_from_offsets_and_the_absent_marker() { let row = json!({"a": "0x10", "b": "0xfffff", "c": "0x100000", "d": ["0x7fff0000", "0x4fb01f0"], "e": 1234567, "f": "0xabcd_not"}); assert_eq!(addresses_of(&row), BTreeSet::from([0x100000, 0x4fb01f0])); } - #[test] - fn nfr3_an_address_inside_a_table_is_named_by_the_table_below_it() { - let symbols: Vec = vec![ - (0x1000, "other".into()), - (0x2000, "??_7Table@@6B@".into()), - (0x2000, "alias".into()), - (0x9000, "far".into()), - ]; - let named = name_addresses(&BTreeSet::from([0x2000, 0x2250, 0x8000]), &symbols); - assert_eq!( - named[&0x2000], - (0, vec!["??_7Table@@6B@".to_owned(), "alias".to_owned()]) - ); - assert_eq!(named[&0x2250].0, 0x250); - assert!(!named.contains_key(&0x8000)); - } - #[test] fn nfr3_x64_lengths_of_real_function_starts() { // Chrome 156.0.8078.74 chip text: the frame setup ends at byte 13. @@ -3075,10 +2703,9 @@ mod tests { } #[test] - fn nfr6_a_build_without_symbols_gets_a_row_from_the_prior_builds_fingerprints() { + fn nfr6_a_new_build_gets_a_row_from_the_prior_rows_fingerprints_alone() { let (row, report) = candidate(&prints_of_prior(), &shifted_code(0x2000, 0x44), "1.0.0.3"); assert!(report.review.is_empty(), "{:?}", report.review); - assert!(report.made_without_pdb); assert_eq!(report.fingerprinted.len(), 4, "{:?}", report.fingerprinted); assert_eq!(row["build"]["toolbar"]["paint"], "0x103300"); assert_eq!(row["frame"]["frame_view_vtable"], 0x103000); @@ -3183,72 +2810,101 @@ mod tests { } #[test] - fn nfr6_a_row_made_without_its_pdb_is_checked_against_the_pdb_row() { - let (mut row, _) = candidate(&prints_of_prior(), &shifted_code(0x2000, 0x44), "1.0.0.3"); - mark_without_pdb(&mut row, "ABC1", "1.0.0.2", "stable", ""); - assert_eq!(row[WITHOUT_PDB]["prior_version"], "1.0.0.2"); - let mut from_pdb = row.clone(); - from_pdb.as_object_mut().unwrap().remove(WITHOUT_PDB); - // A part the fingerprint row left out is no difference. - row["build"]["screen_origin"] = Value::Null; - from_pdb["build"]["screen_origin"] = json!(["0x103400", "5657"]); - let table = json!({"rows": [row]}); - let mut agreed = table.clone(); - assert_eq!( - check_against_pdb(&mut agreed, "B", "1.0.0.3", &from_pdb).unwrap(), - Vec::::new() - ); - assert!(agreed["rows"][0].get(WITHOUT_PDB).is_none()); - // A different address is a difference, and the row keeps its mark. - from_pdb["build"]["toolbar"]["paint"] = json!("0x103310"); - let mut differs = table.clone(); - let differences = check_against_pdb(&mut differs, "B", "1.0.0.3", &from_pdb).unwrap(); - assert_eq!(differences.len(), 1, "{differences:?}"); - assert!( - differences[0].starts_with("build.toolbar.paint:"), - "{differences:?}" - ); - assert_eq!(differs, table); - // A row made from symbols has nothing to check. - let mut plain = json!({"rows": [from_pdb.clone()]}); - assert!(check_against_pdb(&mut plain, "B", "1.0.0.3", &from_pdb).is_err()); + fn nfr3_an_address_that_is_neither_code_nor_a_table_gets_no_fingerprints() { + let bytes = module(&code_refs(&shifted_code(0, 0x10))); + let pe = PeFile::parse(&bytes).unwrap(); + let mut row = verified_row(); + // Inside the code section's range but past what the file holds. + row["build"]["toolbar"]["state"] = json!("0x1f0000"); + let error = add_fingerprints(&mut row, &pe).unwrap_err(); + assert!(error.contains("0x1f0000"), "{error}"); + assert!(row.get("fingerprints").is_none()); } #[test] - fn nfr6_a_row_made_from_a_marked_prior_does_not_inherit_the_mark() { - let mut prior = prior(); - mark_without_pdb(&mut prior, "ABC1", "1.0.0.0", "stable", ""); - let code = new_code(); - let (row, _) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "1.0.0.2"); - assert!(row.get(WITHOUT_PDB).is_none()); + fn nfr3_a_prior_row_without_screen_origin_gives_a_row_without_it() { + let mut row = prior(); + let mut report = PortReport::default(); + leave_out_screen_origin(&mut row, &mut report); + assert!(row["build"]["screen_origin"].is_null()); + assert_eq!(report.left_out.len(), 1); + assert!(report.review.is_empty()); + // A row that has it keeps the place it was found at. + row["build"]["screen_origin"] = json!(["0x101400", "5657"]); + leave_out_screen_origin(&mut row, &mut report); + assert_eq!(row["build"]["screen_origin"][0], "0x101400"); } #[test] - fn nfr6_a_review_line_names_the_prior_function_it_is_about() { - let named = BTreeMap::from([(0x101300, (0x10, vec!["?Paint@V@@".to_owned()]))]); - let mut review = vec![ - "0x101300 matches no place".to_owned(), - "0x101400: no fingerprint".to_owned(), - "the milestone changed".to_owned(), - ]; - name_review_lines(&mut review, &named); + fn nfr6_a_new_row_records_the_class_names_of_the_tables_it_placed() { + let mut prior = verified_row(); + // A name the prior row recorded is not carried: only names confirmed now are written. + prior[CLASSES] = json!({"build.toolbar.stale": "Stale"}); + let classes = BTreeMap::from([(0x101000, "BrowserFrameViewWin".to_owned())]); + let bytes = module(&code_refs(&shifted_code(0x2000, 0x44))); + let pe = PeFile::parse(&bytes).unwrap(); + let (row, report) = + find_candidate(&prior, &prints_of_prior(), None, &classes, "1.0.0.3", &pe); + assert!(report.review.is_empty(), "{:?}", report.review); assert_eq!( - review[0], - "0x101300 matches no place (?Paint@V@@+0x10 in the prior build)" + row[CLASSES], + json!({"frame.frame_view_vtable": "BrowserFrameViewWin"}) ); - assert_eq!(review[1], "0x101400: no fingerprint"); - assert_eq!(review[2], "the milestone changed"); + // With no confirmed name the row records none. + let (row, _) = find_candidate( + &prior, + &prints_of_prior(), + None, + &BTreeMap::new(), + "1.0.0.3", + &pe, + ); + assert!(row.get(CLASSES).is_none()); } #[test] - fn nfr3_an_address_that_is_neither_code_nor_a_table_gets_no_fingerprints() { - let bytes = module(&code_refs(&shifted_code(0, 0x10))); - let pe = PeFile::parse(&bytes).unwrap(); - let mut row = verified_row(); - // Inside the code section's range but past what the file holds. - row["build"]["toolbar"]["state"] = json!("0x1f0000"); - let error = add_fingerprints(&mut row, &pe).unwrap_err(); - assert!(error.contains("0x1f0000"), "{error}"); - assert!(row.get("fingerprints").is_none()); + fn nfr6_every_address_of_a_row_is_known_by_its_path() { + let paths = address_paths(&verified_row()); + assert!(paths.contains(&("frame.frame_view_vtable".to_owned(), 0x101000))); + assert!(paths.contains(&("build.toolbar.paint".to_owned(), 0x101300))); + assert!(paths.contains(&("build.tabs.signatures[0][0]".to_owned(), 0x101300))); + // Offsets and the absent marker are not addresses. + assert!(!paths.iter().any(|(path, _)| path.ends_with("view_layer"))); + assert!(!paths.iter().any(|(path, _)| path.ends_with("missing"))); + } + + #[test] + fn nfr6_a_made_row_is_scored_value_by_value_against_the_verified_row() { + let (made, report) = candidate(&prints_of_prior(), &shifted_code(0x2000, 0x44), "1.0.0.3"); + let mut verified = made.clone(); + let score = score_row(&made, &report, &verified); + assert_eq!(score.wrong(), 0); + assert!(score.features[WINDOW].equal > 0); + // A value found at another place than the verified one is wrong. + verified["build"]["toolbar"]["paint"] = json!("0x103310"); + let score = score_row(&made, &report, &verified); + assert_eq!(score.wrong(), 1, "{:?}", score.features); + let wrong: Vec<&String> = score.features.values().flat_map(|f| &f.different).collect(); + assert!( + wrong[0].starts_with("build.toolbar.paint: made"), + "{wrong:?}" + ); + // A value the made row does not hold is not wrong. + let mut made_less = made.clone(); + made_less["build"]["toolbar"] + .as_object_mut() + .unwrap() + .remove("paint"); + let score = score_row(&made_less, &report, &verified); + assert_eq!(score.wrong(), 0); + // A feature the made row turns off is not compared at all. + let mut off = made.clone(); + off[crate::row_features::SWITCHED_OFF] = json!([{"feature": "split view", "values": []}]); + let mut split = verified.clone(); + split["build"]["split"] = json!({"supported": true, "outline_member": "0x40"}); + off["build"]["split"] = json!({"supported": false, "outline_member": "0x0"}); + let score = score_row(&off, &report, &split); + assert!(score.features["split view"].off); + assert!(score.features["split view"].different.is_empty()); } } diff --git a/rivet/platform/windows/src/row_recipe.rs b/rivet/platform/windows/src/row_recipe.rs index a65f945..ee91f92 100644 --- a/rivet/platform/windows/src/row_recipe.rs +++ b/rivet/platform/windows/src/row_recipe.rs @@ -1,7 +1,7 @@ //! Reads the values of a row that are not addresses from the new build itself. //! -//! A row holds member offsets, table slots and object sizes as well as addresses. They have no -//! public symbols, and a new milestone moves them without notice: Chromium 157 moved many view +//! A row holds member offsets, table slots and object sizes as well as addresses. No name in the +//! module tells them, and a new milestone moves them without notice: Chromium 157 moved many view //! members by 8 bytes and every frame view slot above one removed virtual by one. A port that //! copies them gives a row that looks complete and is wrong. So each such value has recipes that //! read it from the new build's own instructions and tables. A value is taken only when every diff --git a/rivet/platform/windows/src/row_resolve.rs b/rivet/platform/windows/src/row_resolve.rs index 6679072..dd9d19a 100644 --- a/rivet/platform/windows/src/row_resolve.rs +++ b/rivet/platform/windows/src/row_resolve.rs @@ -1,5 +1,5 @@ -//! Finds, in a build whose symbols are not published, the functions and class tables that the -//! prior row's fingerprints do not pin down to one place. +//! Finds, in a new build, the functions and class tables that the prior row's fingerprints do not +//! pin down to one place. //! //! Each such address is looked for by several independent methods, and a place is taken only when //! two different methods name the same place and no method names another. The methods read the @@ -849,7 +849,7 @@ pub enum Kind { pub struct Wanted { pub kind: Kind, pub candidates: Option>, - /// For a class table, the views class the prior build's public symbol names it after. + /// For a class table, the views class whose name its metadata holds in the prior build. pub class: Option, /// For a function a browser command runs first, the command's public id. pub command: Option, diff --git a/rivet/platform/windows/src/row_watch.rs b/rivet/platform/windows/src/row_watch.rs index 842f084..86e7c19 100644 --- a/rivet/platform/windows/src/row_watch.rs +++ b/rivet/platform/windows/src/row_watch.rs @@ -161,7 +161,7 @@ pub fn brave_candidates(releases: &Value) -> Vec { /// What still needs a row, most urgent first: what people have now, then what stable is about to /// serve, then beta, which brings the next milestone weeks ahead and gives a person time to review -/// what symbols cannot show. A version older than every row and not served is history, not news. +/// what the generator cannot read. A version older than every row and not served is history, not news. pub fn pending(candidates: Vec, known: &BTreeSet<(String, String)>) -> Vec { let mut newest_known: BTreeMap<&str, Vec> = BTreeMap::new(); for (browser, version) in known { diff --git a/rivet/runtime/windows/rows/table.json b/rivet/runtime/windows/rows/table.json index 41d56c3..90ba91d 100644 --- a/rivet/runtime/windows/rows/table.json +++ b/rivet/runtime/windows/rows/table.json @@ -2604,17 +2604,5 @@ } ], "schema": 1, - "sequence": 0, - "symbols": { - "Brave 156.1.98.63": "5723B770F8DCB2794C4C44205044422E1", - "Brave 157.1.99.55": "22DC58117C04ED6E4C4C44205044422E1", - "Google Chrome 156.0.8078.74": "713E6CA96EBDD5814C4C44205044422E1", - "Google Chrome 156.0.8078.80": "B2C7C3B72F30D3ED4C4C44205044422E1", - "Google Chrome 156.0.8078.81": "4A4A604319CF7D794C4C44205044422E1", - "Google Chrome 156.0.8078.82": "A28166C00E1AB5384C4C44205044422E1", - "Google Chrome 157.0.8103.38": "B445A620A73B6EA94C4C44205044422E1", - "Google Chrome 157.0.8103.39": "037147ED9738B0F84C4C44205044422E1", - "Microsoft Edge 156.0.4314.50": "F44CB0836ADF26834C4C44205044422E1", - "Microsoft Edge 156.0.4314.58": "03775A411C2857584C4C44205044422E1" - } + "sequence": 0 } diff --git a/rivet/runtime/windows/src/builds/tests.rs b/rivet/runtime/windows/src/builds/tests.rs index 1f70e64..3b4e8cd 100644 --- a/rivet/runtime/windows/src/builds/tests.rs +++ b/rivet/runtime/windows/src/builds/tests.rs @@ -1501,36 +1501,6 @@ fn fr12_edge_58_and_whale_57_record_the_page_view_that_moves_below_the_bar() { } } -#[cfg(windows)] -#[test] -fn nfr6_newest_row_of_each_ported_browser_has_its_symbol_key() { - // The row job ports a new build from the newest row of its browser, read through that row's - // public symbols. A newest row without its symbol key stops every later row of the browser. - let table: serde_json::Value = - serde_json::from_str(CHECKED_IN).expect("the checked-in table is JSON"); - let mut rows = crate::compiled_rows(); - rows.extend(table["rows"].as_array().cloned().unwrap_or_default()); - let version_of = |row: &serde_json::Value| -> Vec { - row["build"]["version"] - .as_array() - .map(|parts| parts.iter().filter_map(serde_json::Value::as_u64).collect()) - .unwrap_or_default() - }; - for browser in ["Google Chrome", "Microsoft Edge", "Brave"] { - let newest = rows - .iter() - .filter(|row| row["build"]["browser"] == browser && !row["frame"].is_null()) - .max_by_key(|row| version_of(row)) - .unwrap_or_else(|| panic!("{browser} has a row")); - let version: Vec = version_of(newest).iter().map(u64::to_string).collect(); - let key = format!("{browser} {}", version.join(".")); - assert!( - table["symbols"][key.as_str()].is_string(), - "{key} has no symbol key in rows/table.json" - ); - } -} - #[cfg(windows)] #[test] fn nfr6_recipes_read_every_row_value_they_cover_again_from_its_own_module() { diff --git a/scripts/check.sh b/scripts/check.sh index 34c08c3..0f97327 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -15,6 +15,7 @@ tests=( scripts/tests/installer.test.sh scripts/tests/windows-features.test.sh scripts/tests/paths.test.sh + scripts/tests/no-pdb.test.sh scripts/tests/layers.test.sh scripts/tests/test-desktop.test.sh scripts/tests/test-window-placement.test.sh diff --git a/scripts/tests/browser-rows.test.sh b/scripts/tests/browser-rows.test.sh index 2e38aa8..111771d 100755 --- a/scripts/tests/browser-rows.test.sh +++ b/scripts/tests/browser-rows.test.sh @@ -25,10 +25,10 @@ expect "proves a row on the real browser" "windows-row-proof.ps1" expect "does not publish a row that needs review" "steps.made.outputs.for_review == 'false'" expect "publishes a signed table" "table-sign" expect "signs with a secret" "secrets.RIVET_TABLE_SIGNING_KEY" -expect "makes a row from fingerprints when the new PDB is missing" "chrome-builds.exe find" -expect "tells a missing PDB from a failed download" "catch [System.IO.FileNotFoundException]" -expect "compares a row made without its PDB once the PDB appears" "chrome-builds.exe pdb-check" -expect "reports a row that differs from its PDB" "Row differs from its PDB" +expect "makes each row from the prior row and the prior build's module" 'chrome-builds.exe find "$temp/prior.json" $env:PRIOR_MODULE' +expect "fingerprints a new row from its own module" "chrome-builds.exe fingerprint" +expect "scores the generator against verified rows" "scripts/tests/row-score.ps1" +expect "a score run makes no row" "github.event.inputs.score == ''" expect "a trial run publishes nothing" "needs.row.result != 'skipped' && !inputs.trial" expect "a trial run proposes nothing" "steps.made.outputs.for_review == 'true' && !inputs.trial" @@ -36,7 +36,7 @@ if [[ -e "$repo_root/.github/workflows/chrome-support.yml" ]]; then echo "chrome-support.yml covers Chrome only and is replaced by browser-rows.yml" >&2 failed=1 fi -for script in fetch-browser fetch-symbols; do +for script in fetch-browser; do if [[ ! -f "$repo_root/.github/scripts/rows/$script.ps1" ]]; then echo "the job needs .github/scripts/rows/$script.ps1" >&2 failed=1 diff --git a/scripts/tests/no-pdb.test.sh b/scripts/tests/no-pdb.test.sh new file mode 100755 index 0000000..2124383 --- /dev/null +++ b/scripts/tests/no-pdb.test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Fails if code, scripts or workflows download or read a browser's debug symbols. Rivet and its row +# generator make every row from the verified prior row, its fingerprints and the two builds' +# modules: a maker can ship a build long before its symbols appear, or never publish them, so +# nothing may wait on them. The rows already verified with symbols stay as the starting point. +set -uo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root" + +# Searches the tracked code, scripts and workflows. A search that cannot run fails, so a broken +# search never passes as a clean one. +search() { + local found status + found="$(git grep -inIE -e "$1" -- rivet scripts .github ':!scripts/tests/no-pdb.test.sh')" + status=$? + if (( status > 1 )); then + echo "error: git grep could not search for $1" >&2 + return 2 + fi + printf '%s' "$found" +} + +# A symbol server, the variable that names one, or the path a PDB is asked for by. +servers='symsrv|msdl\.microsoft\.com|brave-symbols|symbols\.mozilla|/download/symbols|_NT_SYMBOL_PATH|srv\*|\.pdb/|\.dll\.pdb' +found="$(search "$servers")" || exit 2 +if [[ -n "$found" ]]; then + echo "error: no code, script or workflow may name a symbol server:" >&2 + echo "$found" >&2 + exit 1 +fi +echo "ok no code, script or workflow names a symbol server" + +# DbgHelp and DIA read PDBs, a CodeView record names the PDB of a module, and the `pdb` crate parses +# one. Any of them, or a file named after a PDB, is code that reads symbols. +readers='dbghelp|SymInitialize|SymLoadModule|SymEnumSymbols|SymFromName|SymFromAddr|SymSetOptions|msdia|IDiaDataSource|RSDS|CodeView|pdb_key|pdb-key|(^|[^a-z0-9_])pdb([^a-z0-9_]|$)|fetch-symbols' +found="$(search "$readers")" || exit 2 +if [[ -n "$found" ]]; then + echo "error: no code, script or workflow may read a PDB:" >&2 + echo "$found" >&2 + exit 1 +fi +echo "ok no code, script or workflow reads a PDB" diff --git a/scripts/tests/row-port.ps1 b/scripts/tests/row-port.ps1 deleted file mode 100644 index d074f1c..0000000 --- a/scripts/tests/row-port.ps1 +++ /dev/null @@ -1,62 +0,0 @@ -param( - [Parameter(Mandatory = $true)][string]$Rows, - [Parameter(Mandatory = $true)][string]$Generator, - [Parameter(Mandatory = $true)][string]$Symbols -) - -$ErrorActionPreference = 'Stop' -# Proves the row generator on builds people already verified: a row made from the prior build must -# hold the same addresses as the row that was checked by hand. Fields the prior row did not have -# yet are named, since a generator cannot add a feature. Needs the PDBs and an installed browser, so -# a machine without them skips the pair it lacks. -$work = Join-Path ([IO.Path]::GetTempPath()) "rivet-row-port-$PID" -New-Item -ItemType Directory -Force $work | Out-Null - -function Flatten($value, $path, $out) { - if ($value -is [System.Management.Automation.PSCustomObject]) { - foreach ($item in $value.PSObject.Properties) { Flatten $item.Value "$path.$($item.Name)" $out } - } elseif ($value -is [System.Collections.IEnumerable] -and $value -isnot [string]) { - $index = 0 - foreach ($item in $value) { Flatten $item "$path[$index]" $out; $index++ } - } else { - $out[$path] = $value - } -} - -$failed = $false -foreach ($pair in @( - @{ Browser = 'Google Chrome'; Prior = '156.0.8078.48'; New = '156.0.8078.74'; Pdbs = 'chrome-symbols'; Module = 'C:\Program Files\Google\Chrome\Application\156.0.8078.74\chrome.dll' }, - @{ Browser = 'Brave'; Prior = '156.1.98.62'; New = '156.1.98.63'; Pdbs = 'brave-symbols'; Module = 'C:\Program Files\BraveSoftware\Brave-Browser\Application\156.1.98.63\chrome.dll' } -)) { - $old = Join-Path $Symbols "$($pair.Pdbs)\$($pair.Prior)\chrome.dll.pdb" - $new = Join-Path $Symbols "$($pair.Pdbs)\$($pair.New)\chrome.dll.pdb" - if (-not ((Test-Path $old) -and (Test-Path $new) -and (Test-Path $pair.Module))) { - "skip $($pair.Browser) $($pair.New): no symbols or module here" - continue - } - $priorRow = Join-Path $work "prior.json" - $made = Join-Path $work "made.json" - $report = Join-Path $work "report.json" - & $Rows row $pair.Browser $pair.Prior | Set-Content -Encoding utf8 $priorRow - $truth = & $Rows row $pair.Browser $pair.New | ConvertFrom-Json - & $Generator port $priorRow $old $new $pair.Module $pair.New $made $report - $review = (Get-Content -Raw $report | ConvertFrom-Json).review - if ($review.Count -gt 0) { "$($pair.Browser): for review: $review"; $failed = $true } - $a = @{}; $b = @{} - Flatten (Get-Content -Raw $made | ConvertFrom-Json) '' $a - Flatten $truth '' $b - $count = 0 - foreach ($path in $b.Keys) { - if ("$($b[$path])" -notmatch '^0x') { continue } - $count++ - if ($a[$path] -ne $b[$path]) { - if ($a[$path] -eq '0x7fff0000' -or $null -eq $a[$path]) { "$($pair.Browser): $path was added by hand after the prior row"; continue } - "$($pair.Browser): $path made $($a[$path]), verified $($b[$path])" - $failed = $true - } - } - "$($pair.Browser) $($pair.Prior) -> $($pair.New): compared $count address values" -} -Remove-Item -Recurse -Force $work -if ($failed) { throw 'NFR-3: A generated row differs from the row people verified.' } -'NFR-3: Generated rows match the verified rows.' diff --git a/scripts/tests/row-score.ps1 b/scripts/tests/row-score.ps1 new file mode 100644 index 0000000..1e1cd81 --- /dev/null +++ b/scripts/tests/row-score.ps1 @@ -0,0 +1,98 @@ +param( + # rivet-rows.exe, which prints the rows Rivet knows. + [Parameter(Mandatory = $true)][string]$Rows, + # chrome-builds.exe, the row generator. + [Parameter(Mandatory = $true)][string]$Generator, + # The checked-in table, whose rows count as verified like those built into rivet-rows. + [Parameter(Mandatory = $true)][string]$Table, + # Where the modules, rows and scores go. Removed and made again. + [Parameter(Mandatory = $true)][string]$Work, + # `browser@prior>new` pairs. Empty means the list below. + [string[]]$Pairs = @(), + # Fetches each build with .github/scripts/rows/fetch-browser.ps1. Without it, each module is + # taken from `$Modules\\\`. + [switch]$Fetch, + [string]$Modules = '' +) + +$ErrorActionPreference = 'Stop' +# Scores the row generator against answer keys: builds whose rows people verified, made from +# public symbols before Rivet stopped reading them. Each new row is made without symbols from the +# verified row of an older build, the way the row job makes rows, and compared value by value with +# the verified row of the new build. Per feature it reports values found and equal, values found +# but different, and features the made row turns off because their values were not found. +# +# A value found but different is a wrong value, which would reach people's browsers, so any fails +# this script. A feature turned off is not a failure: the runtime keeps it off. A pair whose builds +# cannot be fetched is reported and skipped, and a run that scores no pair fails, since it measured +# nothing. Nothing is published or proposed. +# +# A build newer than every verified row, such as Chrome 158 while the newest verified row is 157, +# has no answer key and cannot be scored here. +if (-not $Pairs -or $Pairs.Count -eq 0) { + $Pairs = @( + 'Google Chrome@156.0.8078.82>157.0.8103.38', + 'Google Chrome@156.0.8078.82>157.0.8103.39', + 'Google Chrome@156.0.8078.74>156.0.8078.80', + 'Microsoft Edge@156.0.4314.50>156.0.4314.58', + 'Brave@156.1.98.63>157.1.99.55' + ) +} + +if (Test-Path $Work) { Remove-Item -Recurse -Force $Work } +New-Item -ItemType Directory -Force $Work | Out-Null + +function Find-BuildModule($browser, $version) { + $dll = if ($browser -eq 'Microsoft Edge') { 'msedge.dll' } else { 'chrome.dll' } + if (-not $Fetch) { + $path = Join-Path $Modules "$browser\$version\$dll" + if (-not (Test-Path $path)) { throw "no module at $path" } + return $path + } + $out = Join-Path $Work ("browser-" + ($browser -replace ' ', '-') + "-$version") + $found = Join-Path $Work ("browser-" + ($browser -replace ' ', '-') + "-$version.txt") + if (-not (Test-Path $found)) { + & "$PSScriptRoot/../../.github/scripts/rows/fetch-browser.ps1" -Browser $browser -Version $version -Out $out -Output $found | Out-Host + } + (Get-Content $found | Where-Object { $_ -like 'module=*' } | Select-Object -First 1) -replace '^module=', '' +} + +function Get-VerifiedRow($browser, $version, $file) { + & $Rows prior $browser $version --table $Table | Set-Content -Encoding utf8 $file + if ($LASTEXITCODE -ne 0) { throw "rivet-rows has no row for $browser at or below $version" } + $row = Get-Content -Raw $file | ConvertFrom-Json + if ($row.frame.version -ne $version) { throw "$browser $version has no verified row; the nearest is $($row.frame.version)" } +} + +$scored = 0 +$wrong = 0 +$failed = @() +foreach ($pair in $Pairs) { + $browser, $versions = $pair -split '@', 2 + $prior, $new = $versions -split '>', 2 + $slug = ($browser -replace ' ', '-') + "-$prior-to-$new" + $priorRow = Join-Path $Work "$slug-prior.json" + $verified = Join-Path $Work "$slug-verified.json" + $score = Join-Path $Work "$slug-score.json" + try { + Get-VerifiedRow $browser $prior $priorRow + Get-VerifiedRow $browser $new $verified + $priorModule = Find-BuildModule $browser $prior + $module = Find-BuildModule $browser $new + } catch { + "skip ${browser} $prior -> ${new}: $_" + continue + } + & $Generator score $priorRow $priorModule $module $verified $score + switch ($LASTEXITCODE) { + 0 { $scored++ } + 5 { $scored++; $wrong++ } + default { $failed += "$browser $prior -> $new (chrome-builds exited with $LASTEXITCODE)" } + } +} + +"scored $scored of $($Pairs.Count) pairs; $wrong with a wrong value" +if ($failed.Count -gt 0) { throw "NFR-6: the generator failed on: $($failed -join '; ')" } +if ($wrong -gt 0) { throw 'NFR-6: a row made without symbols holds a value that differs from its verified row.' } +if ($scored -eq 0) { throw 'NFR-6: no pair could be scored, so nothing was measured.' } +'NFR-6: every value the generator found equals the verified row.'