Fix several patches & newer phpmicro support (#470)

* use upstream phpmicro * move src/global/tests to src/global/ext-tests * move src/global/tests to src/global/ext-tests * prevent file_get_contents memory insufficience * update README * fix libxml >= 2.12 with older PHP (<=8.1) build bug * cleanup code, support newer phpmicro * add --no-strip and --with-upx-pack tests * fix windows sanity check for newer phpmicro * fix windows sanity check for newer phpmicro * test * test * test * update deps for ci
crazywhalecc · Jun 3, 2024 · d258417 · d258417
1 parent 3057d02
commit d258417
Show file tree

Hide file tree

Showing 36 changed files with 742 additions and 71 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -147,7 +147,7 @@ jobs:
 
       # Cache downloaded source
       - id: cache-download
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: downloads
           key: php-${{ matrix.php }}-dependencies
@@ -158,6 +158,19 @@ jobs:
       - name: "Run Build Tests (doctor)"
         run: bin/spc doctor --auto-fix
 
+      - name: "Prepare UPX for Windows"
+        if: matrix.os == 'windows-latest'
+        run: |
+          bin/spc install-pkg upx
+          echo "UPX_CMD=$(php src/globals/test-extensions.php upx)" >> $env:GITHUB_ENV
+
+      - name: "Prepare UPX for Linux"
+        if: matrix.os == 'ubunut-latest'
+        run: |
+          bin/spc install-pkg upx
+          echo "UPX_CMD=$(php src/globals/test-extensions.php upx)" >> $GITHUB_ENV
+
+
       - name: "Run Build Tests (download)"
         uses: nick-fields/retry@v3
         with:
@@ -167,5 +180,10 @@ jobs:
           command: |
             bin/spc download --for-extensions="$(php src/globals/test-extensions.php extensions)" --for-libs="$(php src/globals/test-extensions.php libs)" --with-php=${{ matrix.php }} --ignore-cache-sources=php-src --debug --retry=3
 
-      - name: "Run Build Tests (build)"
-        run: bin/spc build "$(php src/globals/test-extensions.php extensions)" $(php src/globals/test-extensions.php zts) --with-libs="$(php src/globals/test-extensions.php libs)" --build-cli --build-micro --build-fpm --debug
+      - name: "Run Build Tests (build, *nix)"
+        if: matrix.os != 'windows-latest'
+        run: bin/spc build "$(php src/globals/test-extensions.php extensions)" $(php src/globals/test-extensions.php zts) $(php src/globals/test-extensions.php no_strip) $UPX_CMD --with-libs="$(php src/globals/test-extensions.php libs)" --build-cli --build-micro --build-fpm --debug
+
+      - name: "Run Build Tests (build, windows)"
+        if: matrix.os == 'windows-latest'
+        run: bin/spc build "$(php src/globals/test-extensions.php extensions)" $(php src/globals/test-extensions.php zts) $(php src/globals/test-extensions.php no_strip) $env:UPX_CMD --with-libs="$(php src/globals/test-extensions.php libs)" --build-cli --build-micro --build-fpm --debug
diff --git a/README-zh.md b/README-zh.md
@@ -74,7 +74,7 @@ static-php-cli（简称 `spc`）有许多特性：
 
 当前支持编译的 PHP 版本：
 
-> :warning: 支持，但可能不再提供修复
+> :warning: 支持，但 static-php-cli 作者可能不再提供补丁修复
 > 
 > :heavy_check_mark: 支持
 > 
@@ -86,7 +86,7 @@ static-php-cli（简称 `spc`）有许多特性：
 | 7.3         | :warning:          | phpmicro 和许多扩展不支持 7.3、7.4 版本 |
 | 7.4         | :warning:          | phpmicro 和许多扩展不支持 7.3、7.4 版本 |
 | 8.0         | :heavy_check_mark: | PHP 官方已停止 8.0 的维护            |
-| 8.1         | :heavy_check_mark: |                              |
+| 8.1         | :heavy_check_mark: | PHP 官方仅对 8.1 提供安全更新          |
 | 8.2         | :heavy_check_mark: |                              |
 | 8.3         | :heavy_check_mark: |                              |
 

diff --git a/README.md b/README.md
@@ -82,7 +82,7 @@ Here is the supported OS and arch, where :octocat: represents support for GitHub
 
 Currently supported PHP versions for compilation: 
 
-> :warning: supported but not maintained
+> :warning: supported but not maintained by static-php-cli authors
 > 
 > :heavy_check_mark: supported
 > 
@@ -94,7 +94,7 @@ Currently supported PHP versions for compilation:
 | 7.3         | :warning:          | phpmicro and some extensions not supported on 7.x |
 | 7.4         | :warning:          | phpmicro and some extensions not supported on 7.x |
 | 8.0         | :heavy_check_mark: | PHP official has stopped maintenance of 8.0       |
-| 8.1         | :heavy_check_mark: |                                                   |
+| 8.1         | :heavy_check_mark: | PHP official has security fixes only              |
 | 8.2         | :heavy_check_mark: |                                                   |
 | 8.3         | :heavy_check_mark: |                                                   |
 

diff --git a/config/source.json b/config/source.json
@@ -463,7 +463,7 @@
         "type": "git",
         "path": "php-src/sapi/micro",
         "rev": "master",
-        "url": "https://github.com/static-php/phpmicro",
+        "url": "https://github.com/easysoft/phpmicro",
         "license": {
             "type": "file",
             "path": "LICENSE"

diff --git a/phpstan.neon b/phpstan.neon
@@ -13,6 +13,6 @@ parameters:
     - PHP_OS_FAMILY
   excludePaths:
       analyseAndScan:
-            - ./src/globals/tests/swoole.php
-            - ./src/globals/tests/swoole.phpt
+            - ./src/globals/ext-tests/swoole.php
+            - ./src/globals/ext-tests/swoole.phpt
             - ./src/globals/test-extensions.php
diff --git a/src/SPC/builder/BuilderBase.php b/src/SPC/builder/BuilderBase.php
@@ -9,6 +9,7 @@
 use SPC\exception\RuntimeException;
 use SPC\exception\WrongUsageException;
 use SPC\store\Config;
+use SPC\store\FileSystem;
 use SPC\store\SourceManager;
 use SPC\util\CustomExt;
 
@@ -275,6 +276,24 @@ public function getPHPVersionFromArchive(?string $file = null): false|string
         return false;
     }
 
+    public function getMicroVersion(): false|string
+    {
+        $file = FileSystem::convertPath(SOURCE_PATH . '/php-src/sapi/micro/php_micro.h');
+        if (!file_exists($file)) {
+            return false;
+        }
+
+        $content = file_get_contents($file);
+        $ver = '';
+        preg_match('/#define PHP_MICRO_VER_MAJ (\d)/m', $content, $match);
+        $ver .= $match[1] . '.';
+        preg_match('/#define PHP_MICRO_VER_MIN (\d)/m', $content, $match);
+        $ver .= $match[1] . '.';
+        preg_match('/#define PHP_MICRO_VER_PAT (\d)/m', $content, $match);
+        $ver .= $match[1];
+        return $ver;
+    }
+
     /**
      * Get build type name string to display.
      *
@@ -434,4 +453,26 @@ protected function generateMicroExtTests(): string
         $php .= "echo '[micro-test-end]';\n";
         return $php;
     }
+
+    protected function getMicroTestTasks(): array
+    {
+        return [
+            'micro_ext_test' => [
+                'content' => ($this->getOption('without-micro-ext-test') ? '<?php echo "[micro-test-start][micro-test-end]";' : $this->generateMicroExtTests()),
+                'conditions' => [
+                    function ($ret) { return $ret === 0; },
+                    function ($ret, $out) {
+                        $raw_out = trim(implode('', $out));
+                        return str_starts_with($raw_out, '[micro-test-start]') && str_ends_with($raw_out, '[micro-test-end]');
+                    },
+                ],
+            ],
+            'micro_zend_bug_test' => [
+                'content' => ($this->getOption('without-micro-ext-test') ? '<?php echo "hello";' : file_get_contents(ROOT_DIR . '/src/globals/common-tests/micro_zend_mm_heap_corrupted.txt')),
+                'conditions' => [
+                    function ($ret) { return $ret === 0; },
+                ],
+            ],
+        ];
+    }
 }
diff --git a/src/SPC/builder/BuilderProvider.php b/src/SPC/builder/BuilderProvider.php
@@ -27,6 +27,8 @@ class BuilderProvider
      */
     public static function makeBuilderByInput(InputInterface $input): BuilderBase
     {
+        ini_set('memory_limit', '2G');
+
         self::$builder = match (PHP_OS_FAMILY) {
             'Windows' => new WindowsBuilder($input->getOptions()),
             'Darwin' => new MacOSBuilder($input->getOptions()),

diff --git a/src/SPC/builder/Extension.php b/src/SPC/builder/Extension.php
@@ -170,19 +170,19 @@ public function patchBeforeMake(): bool
     public function runCliCheckUnix(): void
     {
         // Run compile check if build target is cli
-        // If you need to run some check, overwrite this or add your assert in src/globals/tests/{extension_name}.php
+        // If you need to run some check, overwrite this or add your assert in src/globals/ext-tests/{extension_name}.php
         // If check failed, throw RuntimeException
         [$ret] = shell()->execWithResult(BUILD_ROOT_PATH . '/bin/php --ri "' . $this->getDistName() . '"', false);
         if ($ret !== 0) {
             throw new RuntimeException('extension ' . $this->getName() . ' failed compile check: php-cli returned ' . $ret);
         }
 
-        if (file_exists(ROOT_DIR . '/src/globals/tests/' . $this->getName() . '.php')) {
+        if (file_exists(ROOT_DIR . '/src/globals/ext-tests/' . $this->getName() . '.php')) {
             // Trim additional content & escape special characters to allow inline usage
             $test = str_replace(
                 ['<?php', 'declare(strict_types=1);', "\n", '"', '$'],
                 ['', '', '', '\"', '\$'],
-                file_get_contents(ROOT_DIR . '/src/globals/tests/' . $this->getName() . '.php')
+                file_get_contents(ROOT_DIR . '/src/globals/ext-tests/' . $this->getName() . '.php')
             );
 
             [$ret, $out] = shell()->execWithResult(BUILD_ROOT_PATH . '/bin/php -r "' . trim($test) . '"');
@@ -201,19 +201,19 @@ public function runCliCheckUnix(): void
     public function runCliCheckWindows(): void
     {
         // Run compile check if build target is cli
-        // If you need to run some check, overwrite this or add your assert in src/globals/tests/{extension_name}.php
+        // If you need to run some check, overwrite this or add your assert in src/globals/ext-tests/{extension_name}.php
         // If check failed, throw RuntimeException
         [$ret] = cmd()->execWithResult(BUILD_ROOT_PATH . '/bin/php.exe --ri "' . $this->getDistName() . '"', false);
         if ($ret !== 0) {
             throw new RuntimeException('extension ' . $this->getName() . ' failed compile check: php-cli returned ' . $ret);
         }
 
-        if (file_exists(FileSystem::convertPath(ROOT_DIR . '/src/globals/tests/' . $this->getName() . '.php'))) {
+        if (file_exists(FileSystem::convertPath(ROOT_DIR . '/src/globals/ext-tests/' . $this->getName() . '.php'))) {
             // Trim additional content & escape special characters to allow inline usage
             $test = str_replace(
                 ['<?php', 'declare(strict_types=1);', "\n", '"', '$'],
                 ['', '', '', '\"', '$'],
-                file_get_contents(FileSystem::convertPath(ROOT_DIR . '/src/globals/tests/' . $this->getName() . '.php'))
+                file_get_contents(FileSystem::convertPath(ROOT_DIR . '/src/globals/ext-tests/' . $this->getName() . '.php'))
             );
 
             [$ret] = cmd()->execWithResult(BUILD_ROOT_PATH . '/bin/php.exe -r "' . trim($test) . '"');

diff --git a/src/SPC/builder/freebsd/BSDBuilder.php b/src/SPC/builder/freebsd/BSDBuilder.php
@@ -181,7 +181,7 @@ protected function buildMicro(): void
         }
         if ($this->getExt('phar')) {
             $this->phar_patched = true;
-            SourcePatcher::patchMicro(['phar']);
+            SourcePatcher::patchMicroPhar($this->getPHPVersionID());
         }
 
         $enable_fake_cli = $this->getOption('with-micro-fake-cli', false) ? ' -DPHP_MICRO_FAKE_CLI' : '';
@@ -202,7 +202,7 @@ protected function buildMicro(): void
         $this->deployBinary(BUILD_TARGET_MICRO);
 
         if ($this->phar_patched) {
-            SourcePatcher::patchMicro(['phar'], true);
+            SourcePatcher::unpatchMicroPhar();
         }
     }
 

diff --git a/src/SPC/builder/linux/LinuxBuilder.php b/src/SPC/builder/linux/LinuxBuilder.php
@@ -149,7 +149,11 @@ public function buildPHP(int $build_target = BUILD_TARGET_NONE): void
 
         // process micro upx patch if micro sapi enabled
         if ($enable_micro) {
-            $this->processMicroUPX();
+            if (version_compare($this->getMicroVersion(), '0.2.0') < 0) {
+                // for phpmicro 0.1.x
+                $this->processMicroUPXLegacy();
+            }
+            // micro latest needs do strip and upx pack later (strip, upx, cut binary manually supported)
         }
 
         shell()->cd(SOURCE_PATH . '/php-src')
@@ -236,7 +240,7 @@ protected function buildMicro(): void
         }
         if ($this->getExt('phar')) {
             $this->phar_patched = true;
-            SourcePatcher::patchMicro(['phar']);
+            SourcePatcher::patchMicroPhar($this->getPHPVersionID());
         }
 
         $enable_fake_cli = $this->getOption('with-micro-fake-cli', false) ? ' -DPHP_MICRO_FAKE_CLI' : '';
@@ -250,10 +254,12 @@ protected function buildMicro(): void
             ->exec('sed -i "s|//lib|/lib|g" Makefile')
             ->exec("\$SPC_CMD_PREFIX_PHP_MAKE {$vars} micro");
 
+        $this->processMicroUPX();
+
         $this->deployBinary(BUILD_TARGET_MICRO);
 
         if ($this->phar_patched) {
-            SourcePatcher::patchMicro(['phar'], true);
+            SourcePatcher::unpatchMicroPhar();
         }
     }
 
@@ -304,11 +310,11 @@ private function getMakeExtraVars(): array
     }
 
     /**
-     * Apply option --no-strip and --with-upx-pack for micro sapi.
+     * Apply option --no-strip and --with-upx-pack for micro sapi (only for phpmicro 0.1.x)
      *
      * @throws FileSystemException
      */
-    private function processMicroUPX(): void
+    private function processMicroUPXLegacy(): void
     {
         // upx pack and strip for micro
         // but always restore Makefile.frag.bak first
@@ -346,4 +352,26 @@ private function processMicroUPX(): void
             );
         }
     }
+
+    private function processMicroUPX(): void
+    {
+        if (version_compare($this->getMicroVersion(), '0.2.0') >= 0 && !$this->getOption('no-strip', false)) {
+            shell()->exec('strip --strip-all ' . SOURCE_PATH . '/php-src/sapi/micro/micro.sfx');
+
+            if ($this->getOption('with-upx-pack')) {
+                // strip first
+                shell()->exec(getenv('UPX_EXEC') . ' --best ' . SOURCE_PATH . '/php-src/sapi/micro/micro.sfx');
+                // cut binary with readelf
+                [$ret, $out] = shell()->execWithResult('readelf -l ' . SOURCE_PATH . '/php-src/sapi/micro/micro.sfx | awk \'/LOAD|GNU_STACK/ {getline; print $1, $2, $3, $4, $6, $7}\'');
+                $out[1] = explode(' ', $out[1]);
+                $offset = $out[1][0];
+                if ($ret !== 0 || !str_starts_with($offset, '0x')) {
+                    throw new RuntimeException('Cannot find offset in readelf output');
+                }
+                $offset = hexdec($offset);
+                // remove upx extra wastes
+                file_put_contents(SOURCE_PATH . '/php-src/sapi/micro/micro.sfx', substr(file_get_contents(SOURCE_PATH . '/php-src/sapi/micro/micro.sfx'), 0, $offset));
+            }
+        }
+    }
 }
diff --git a/src/SPC/builder/macos/MacOSBuilder.php b/src/SPC/builder/macos/MacOSBuilder.php
@@ -233,7 +233,7 @@ protected function buildMicro(): void
         }
         if ($this->getExt('phar')) {
             $this->phar_patched = true;
-            SourcePatcher::patchMicro(['phar']);
+            SourcePatcher::patchMicroPhar($this->getPHPVersionID());
         }
 
         $enable_fake_cli = $this->getOption('with-micro-fake-cli', false) ? ' -DPHP_MICRO_FAKE_CLI' : '';
@@ -251,7 +251,7 @@ protected function buildMicro(): void
         $this->deployBinary(BUILD_TARGET_MICRO);
 
         if ($this->phar_patched) {
-            SourcePatcher::patchMicro(['phar'], true);
+            SourcePatcher::unpatchMicroPhar();
         }
     }
 

diff --git a/src/SPC/builder/unix/UnixBuilderBase.php b/src/SPC/builder/unix/UnixBuilderBase.php
@@ -162,22 +162,20 @@ protected function sanityCheck(int $build_target): void
 
         // sanity check for phpmicro
         if (($build_target & BUILD_TARGET_MICRO) === BUILD_TARGET_MICRO) {
-            if (file_exists(SOURCE_PATH . '/hello.exe')) {
-                @unlink(SOURCE_PATH . '/hello.exe');
-            }
-            file_put_contents(
-                SOURCE_PATH . '/hello.exe',
-                file_get_contents(SOURCE_PATH . '/php-src/sapi/micro/micro.sfx') .
-                ($this->getOption('without-micro-ext-test') ? '<?php echo "[micro-test-start][micro-test-end]";' : $this->generateMicroExtTests())
-            );
-            chmod(SOURCE_PATH . '/hello.exe', 0755);
-            [$ret, $output2] = shell()->execWithResult(SOURCE_PATH . '/hello.exe');
-            $raw_out = trim(implode('', $output2));
-            $condition[0] = $ret === 0;
-            $condition[1] = str_starts_with($raw_out, '[micro-test-start]') && str_ends_with($raw_out, '[micro-test-end]');
-            foreach ($condition as $k => $v) {
-                if (!$v) {
-                    throw new RuntimeException("micro failed sanity check with condition[{$k}], ret[{$ret}], out[{$raw_out}]");
+            $test_task = $this->getMicroTestTasks();
+            foreach ($test_task as $task_name => $task) {
+                $test_file = SOURCE_PATH . '/' . $task_name . '.exe';
+                if (file_exists($test_file)) {
+                    @unlink($test_file);
+                }
+                file_put_contents($test_file, file_get_contents(SOURCE_PATH . '/php-src/sapi/micro/micro.sfx') . $task['content']);
+                chmod($test_file, 0755);
+                [$ret, $out] = shell()->execWithResult($test_file);
+                foreach ($task['conditions'] as $condition => $closure) {
+                    if (!$closure($ret, $out)) {
+                        $raw_out = trim(implode('', $out));
+                        throw new RuntimeException("micro failed sanity check: {$task_name}, condition [{$condition}], ret[{$ret}], out[{$raw_out}]");
+                    }
                 }
             }
         }