Add benchmarking to CI

Closes GH-11068
3 years ago · afbc71d88d
6 changed files with 356 additions and 0 deletions
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@ -215,3 +215,95 @@ jobs:
        run: .github/scripts/windows/build.bat
      - name: Test
        run: .github/scripts/windows/test.bat
+  BENCHMARKING:
+    name: BENCHMARKING
+    runs-on: ubuntu-22.04
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: apt
+        run: |
+          set -x
+          sudo apt-get update
+          sudo apt-get install \
+            bison \
+            libonig-dev \
+            libsqlite3-dev \
+            openssl \
+            re2c \
+            valgrind
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: "${{github.job}}-${{hashFiles('main/php_version.h')}}"
+          append-timestamp: false
+      - name: ./configure
+        run: |
+          set -x
+          ./buildconf --force
+          ./configure \
+            --disable-debug \
+            --enable-mbstring \
+            --enable-opcache \
+            --enable-option-checking=fatal \
+            --enable-sockets \
+            --enable-werror \
+            --prefix=/usr \
+            --with-config-file-scan-dir=/etc/php.d \
+            --with-mysqli=mysqlnd \
+            --with-openssl \
+            --with-pdo-sqlite \
+            --with-valgrind
+      - name: make
+        run: make -j$(/usr/bin/nproc) >/dev/null
+      - name: make install
+        run: |
+          set -x
+          sudo make install
+          sudo mkdir -p /etc/php.d
+          sudo chmod 777 /etc/php.d
+          echo mysqli.default_socket=/var/run/mysqld/mysqld.sock > /etc/php.d/mysqli.ini
+          echo zend_extension=opcache.so >> /etc/php.d/opcache.ini
+          echo opcache.enable=1 >> /etc/php.d/opcache.ini
+          echo opcache.enable_cli=1 >> /etc/php.d/opcache.ini
+      - name: Setup
+        run: |
+          git config --global user.name "Benchmark"
+          git config --global user.email "benchmark@php.net"
+          sudo service mysql start
+          mysql -uroot -proot -e "CREATE DATABASE IF NOT EXISTS wordpress"
+          mysql -uroot -proot -e "CREATE USER 'wordpress'@'localhost' IDENTIFIED BY 'wordpress'; FLUSH PRIVILEGES;"
+          mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* TO 'wordpress'@'localhost' WITH GRANT OPTION;"
+      - name: git checkout benchmarking-data
+        uses: actions/checkout@v3
+        with:
+          repository: php/benchmarking-data
+          ssh-key: ${{ secrets.BENCHMARKING_DATA_DEPLOY_KEY }}
+          path: benchmark/repos/data
+      - name: Benchmark
+        run: php benchmark/benchmark.php true
+      - name: Store result
+        if: github.event_name == 'push'
+        run: |
+          set -x
+          cd benchmark/repos/data
+          git pull --autostash
+          if [ -e ".git/MERGE_HEAD" ]; then
+            echo "Merging, can't proceed"
+            exit 1
+          fi
+          git add .
+          if git diff --cached --quiet; then
+            exit 0
+          fi
+          git commit -m "Add result for ${{ github.repository }}@${{ steps.determine-commit.outputs.sha }}"
+          git push
+      - name: Show diff
+        if: github.event_name == 'pull_request'
+        run: >-
+          php benchmark/generate_diff.php
+          ${{ github.event.pull_request.head.sha }}
+          $(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})
+          > $GITHUB_STEP_SUMMARY
--- a/benchmark/.gitignore
+++ b/benchmark/.gitignore
@ -0,0 +1 @@
+/repos
--- a/benchmark/benchmark.php
+++ b/benchmark/benchmark.php
@ -0,0 +1,117 @@
+<?php
+
+require_once __DIR__ . '/shared.php';
+
+$storeResult = ($argv[1] ?? 'false') === 'true';
+$phpCgi = $argv[2] ?? dirname(PHP_BINARY) . '/php-cgi';
+if (!file_exists($phpCgi)) {
+    fwrite(STDERR, "php-cgi not found\n");
+    exit(1);
+}
+
+function main() {
+    global $storeResult;
+
+    $data = [];
+    $data['Zend/bench.php'] = runBench(false);
+    $data['Zend/bench.php JIT'] = runBench(true);
+    $data['Symfony Demo 2.2.3'] = runSymfonyDemo(false);
+    $data['Symfony Demo 2.2.3 JIT'] = runSymfonyDemo(true);
+    $data['Wordpress 6.2'] = runWordpress(false);
+    $data['Wordpress 6.2 JIT'] = runWordpress(true);
+    $result = json_encode($data, JSON_PRETTY_PRINT) . "\n";
+
+    fwrite(STDOUT, $result);
+
+    if ($storeResult) {
+        storeResult($result);
+    }
+}
+
+function storeResult(string $result) {
+    $repo = __DIR__ . '/repos/data';
+    cloneRepo($repo, 'git@github.com:php/benchmarking-data.git');
+
+    $commitHash = getPhpSrcCommitHash();
+    $dir = $repo . '/' . substr($commitHash, 0, 2) . '/' . $commitHash;
+    $summaryFile = $dir . '/summary.json';
+    if (!is_dir($dir)) {
+        mkdir($dir, 0755, true);
+    }
+    file_put_contents($summaryFile, $result);
+}
+
+function getPhpSrcCommitHash(): string {
+    $result = runCommand(['git', 'log', '--pretty=format:%H', '-n', '1'], dirname(__DIR__));
+    return $result->stdout;
+}
+
+function runBench(bool $jit): array {
+    return runValgrindPhpCgiCommand([dirname(__DIR__) . '/Zend/bench.php'], jit: $jit);
+}
+
+function runSymfonyDemo(bool $jit): array {
+    $dir = __DIR__ . '/repos/symfony-demo-2.2.3';
+    cloneRepo($dir, 'https://github.com/php/benchmarking-symfony-demo-2.2.3.git');
+    runPhpCommand([$dir . '/bin/console', 'cache:clear']);
+    runPhpCommand([$dir . '/bin/console', 'cache:warmup']);
+    return runValgrindPhpCgiCommand([$dir . '/public/index.php'], cwd: $dir, jit: $jit, warmup: 50);
+}
+
+function runWordpress(bool $jit): array {
+    $dir = __DIR__ . '/repos/wordpress-6.2';
+    cloneRepo($dir, 'https://github.com/php/benchmarking-wordpress-6.2.git');
+
+    /* FIXME: It might be better to use a stable version of PHP for this command because we can't
+     * easily alter the phar file */
+    runPhpCommand([
+        '-d error_reporting=0',
+        'wp-cli.phar', 
+        'core',
+        'install',
+        '--url=wordpress.local',
+        '--title="Wordpress"',
+        '--admin_user=wordpress',
+        '--admin_password=wordpress',
+        '--admin_email=benchmark@php.net',
+    ], $dir);
+
+    // Warmup
+    runPhpCommand([$dir . '/index.php'], $dir);
+    return runValgrindPhpCgiCommand([$dir . '/index.php'], cwd: $dir, jit: $jit, warmup: 50);
+}
+
+function runPhpCommand(array $args, ?string $cwd = null): ProcessResult {
+    return runCommand([PHP_BINARY, ...$args], $cwd);
+}
+
+function runValgrindPhpCgiCommand(
+    array $args,
+    ?string $cwd = null,
+    bool $jit = false,
+    int $warmup = 0,
+): array {
+    global $phpCgi;
+    $process = runCommand([
+        'valgrind',
+        '--tool=callgrind',
+        '--dump-instr=yes',
+        '--callgrind-out-file=/dev/null',
+        '--',
+        $phpCgi,
+        '-T' . ($warmup ? $warmup . ',' : '') . '1',
+        '-d max_execution_time=0',
+        '-d opcache.enable=1',
+        '-d opcache.jit_buffer_size=' . ($jit ? '128M' : '0'),
+        ...$args,
+    ]);
+    $instructions = extractInstructionsFromValgrindOutput($process->stderr);
+    return ['instructions' => $instructions];
+}
+
+function extractInstructionsFromValgrindOutput(string $output): string {
+    preg_match("(==[0-9]+== Events    : Ir\n==[0-9]+== Collected : (?<instructions>[0-9]+))", $output, $matches);
+    return $matches['instructions'] ?? throw new \Exception('Unexpected valgrind output');
+}
+
+main();
--- a/benchmark/docker-compose.yml
+++ b/benchmark/docker-compose.yml
@ -0,0 +1,11 @@
+version: "3.8"
+services:
+  wordpress_db:
+    image: mysql:8.0
+    ports:
+      - "3306:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+      MYSQL_DATABASE: wordpress
+      MYSQL_USER: wordpress
+      MYSQL_PASSWORD: wordpress
--- a/benchmark/generate_diff.php
+++ b/benchmark/generate_diff.php
@ -0,0 +1,63 @@
+<?php
+
+require_once __DIR__ . '/shared.php';
+
+function main(?string $headCommitHash, ?string $baseCommitHash) {
+    if ($headCommitHash === null || $baseCommitHash === null) {
+        fwrite(STDERR, "Usage: php generate_diff.php HEAD_COMMIT_HASH BASE_COMMIT_HASH\n");
+        exit(1);
+    }
+
+    $repo = __DIR__ . '/repos/data';
+    cloneRepo($repo, 'git@github.com:php/benchmarking-data.git');
+    $headSummaryFile = $repo . '/' . substr($headCommitHash, 0, 2) . '/' . $headCommitHash . '/summary.json';
+    $baseSummaryFile = $repo . '/' . substr($baseCommitHash, 0, 2) . '/' . $baseCommitHash . '/summary.json';
+    if (!file_exists($headSummaryFile)) {
+        return "Head commit '$headCommitHash' not found\n";
+    }
+    if (!file_exists($baseSummaryFile)) {
+        return "Base commit '$baseCommitHash' not found\n";
+    }
+    $headSummary  = json_decode(file_get_contents($headSummaryFile), true);
+    $baseSummary  = json_decode(file_get_contents($baseSummaryFile), true);
+
+    $headCommitHashShort = substr($headCommitHash, 0, 7);
+    $baseCommitHashShort = substr($baseCommitHash, 0, 7);
+    $output = "| Benchmark | Base ($baseCommitHashShort) | Head ($headCommitHashShort) | Diff |\n";
+    $output .= "|---|---|---|---|\n";
+    foreach ($headSummary as $name => $headBenchmark) {
+        $baseInstructions = $baseSummary[$name]['instructions'] ?? null;
+        $headInstructions = $headSummary[$name]['instructions'];
+        $output .= "| $name | "
+            . formatInstructions($baseInstructions) . " | "
+            . formatInstructions($headInstructions) . " | "
+            . formatDiff($baseInstructions, $headInstructions) . " |\n";
+    }
+    return $output;
+}
+
+function formatInstructions(?int $instructions): string {
+    if ($instructions === null) {
+        return '-';
+    }
+    if ($instructions > 1e6) {
+        return sprintf('%.0fM', $instructions / 1e6);
+    } elseif ($instructions > 1e3) {
+        return sprintf('%.0fK', $instructions / 1e3);
+    } else {
+        return (string) $instructions;
+    }
+}
+
+function formatDiff(?int $baseInstructions, int $headInstructions): string {
+    if ($baseInstructions === null) {
+        return '-';
+    }
+    $instructionDiff = $headInstructions - $baseInstructions;
+    return sprintf('%.2f%%', $instructionDiff / $baseInstructions * 100);
+}
+
+$headCommitHash = $argv[1] ?? null;
+$baseCommitHash = $argv[2] ?? null;
+$output = main($headCommitHash, $baseCommitHash);
+fwrite(STDOUT, $output);
--- a/benchmark/shared.php
+++ b/benchmark/shared.php
@ -0,0 +1,72 @@
+<?php
+
+class ProcessResult {
+    public $stdout;
+    public $stderr;
+}
+
+function runCommand(array $args, ?string $cwd = null): ProcessResult {
+    $cmd = implode(' ', array_map('escapeshellarg', $args));
+    $pipes = null;
+    $result = new ProcessResult();
+    $descriptorSpec = [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']];
+    fwrite(STDOUT, "> $cmd\n");
+    $processHandle = proc_open($cmd, $descriptorSpec, $pipes, $cwd ?? getcwd(), null);
+
+    $stdin = $pipes[0];
+    $stdout = $pipes[1];
+    $stderr = $pipes[2];
+
+    fclose($stdin);
+
+    stream_set_blocking($stdout, false);
+    stream_set_blocking($stderr, false);
+
+    $stdoutEof = false;
+    $stderrEof = false;
+
+    do {
+        $read = [$stdout, $stderr];
+        $write = null;
+        $except = null;
+
+        stream_select($read, $write, $except, 1, 0);
+
+        foreach ($read as $stream) {
+            $chunk = fgets($stream);
+            if ($stream === $stdout) {
+                $result->stdout .= $chunk;
+            } elseif ($stream === $stderr) {
+                $result->stderr .= $chunk;
+            }
+        }
+
+        $stdoutEof = $stdoutEof || feof($stdout);
+        $stderrEof = $stderrEof || feof($stderr);
+    } while(!$stdoutEof || !$stderrEof);
+
+    fclose($stdout);
+    fclose($stderr);
+
+    $statusCode = proc_close($processHandle);
+    if ($statusCode !== 0) {
+        fwrite(STDOUT, $result->stdout);
+        fwrite(STDERR, $result->stderr);
+        fwrite(STDERR, 'Exited with status code ' . $statusCode . "\n");
+        exit($statusCode);
+    }
+
+    return $result;
+}
+
+function cloneRepo(string $path, string $url) {
+    if (is_dir($path)) {
+        return;
+    }
+    $dir = dirname($path);
+    $repo = basename($path);
+    if (!is_dir($dir)) {
+        mkdir($dir, 0755, true);
+    }
+    runCommand(['git', 'clone', '-q', '--end-of-options', $url, $repo], dirname($path));
+}