More detailed logging&improvements

danog · Sep 3, 2023 · f79012d · f79012d
1 parent 535f1b7
commit f79012d
Show file tree

Hide file tree

Showing 13 changed files with 93 additions and 99 deletions.
diff --git a/src/Psalm/Codebase.php b/src/Psalm/Codebase.php
@@ -2,7 +2,6 @@
 
 namespace Psalm;
 
-use Amp\Parallel\Worker\WorkerPool;
 use Exception;
 use InvalidArgumentException;
 use LanguageServerProtocol\Command;
@@ -38,7 +37,6 @@
 use Psalm\Internal\Codebase\TaintFlowGraph;
 use Psalm\Internal\DataFlow\TaintSink;
 use Psalm\Internal\DataFlow\TaintSource;
-use Psalm\Internal\Fork\Pool;
 use Psalm\Internal\LanguageServer\PHPMarkdownContent;
 use Psalm\Internal\LanguageServer\Reference;
 use Psalm\Internal\MethodIdentifier;
@@ -98,6 +96,7 @@
 
 use const PHP_VERSION_ID;
 
+/** @psalm-import-type PoolData from Scanner */
 final class Codebase
 {
     /**
@@ -394,6 +393,43 @@ private function loadAnalyzer(): void
         );
     }
 
+    /**
+     * @internal
+     * @param PoolData $pool_data
+     */
+    public function addThreadData(array $pool_data): void
+    {
+        IssueBuffer::addIssues($pool_data['issues']);
+
+        $this->statements_provider->addChangedMembers(
+            $pool_data['changed_members'],
+        );
+        $this->statements_provider->addUnchangedSignatureMembers(
+            $pool_data['unchanged_signature_members'],
+        );
+        $this->statements_provider->addDiffMap(
+            $pool_data['diff_map'],
+        );
+        $this->statements_provider->addDeletionRanges(
+            $pool_data['deletion_ranges'],
+        );
+        $this->statements_provider->addErrors($pool_data['errors']);
+
+        if ($this->taint_flow_graph && $pool_data['taint_data']) {
+            $this->taint_flow_graph->addGraph($pool_data['taint_data']);
+        }
+
+        $this->file_storage_provider->addMore($pool_data['file_storage']);
+        $this->classlike_storage_provider->addMore($pool_data['classlike_storage']);
+
+        $this->classlikes->addThreadData($pool_data['classlikes_data']);
+
+        if ($this->statements_provider->parser_cache_provider) {
+            $this->statements_provider->parser_cache_provider->addNewFileContentHashes(
+                $pool_data['new_file_content_hashes'],
+            );
+        }
+    }
     /**
      * @param array<string> $candidate_files
      */

diff --git a/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php b/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php
@@ -69,6 +69,7 @@
 use function explode;
 use function file_exists;
 use function fwrite;
+use function get_object_vars;
 use function implode;
 use function in_array;
 use function is_dir;
@@ -77,7 +78,6 @@
 use function mkdir;
 use function number_format;
 use function preg_match;
-use function Psl\Type\array_key;
 use function rename;
 use function sprintf;
 use function strlen;

diff --git a/src/Psalm/Internal/Codebase/Analyzer.php b/src/Psalm/Internal/Codebase/Analyzer.php
@@ -16,8 +16,6 @@
 use Psalm\Internal\FileManipulation\FunctionDocblockManipulator;
 use Psalm\Internal\FileManipulation\PropertyDocblockManipulator;
 use Psalm\Internal\Fork\AnalyzerTask;
-use Psalm\Internal\Fork\InitAnalyzerTask;
-use Psalm\Internal\Fork\Pool;
 use Psalm\Internal\Fork\ShutdownAnalyzerTask;
 use Psalm\Internal\Provider\FileProvider;
 use Psalm\Internal\Provider\FileStorageProvider;
@@ -337,14 +335,14 @@ private function doAnalysis(ProjectAnalyzer $project_analyzer): void
 
             $this->progress->debug('Forking analysis' . "\n");
 
-            $forked_pool_data = $project_analyzer->pool->run(
+            $project_analyzer->pool->run(
                 $new_file_paths,
-                new InitAnalyzerTask(),
                 AnalyzerTask::class,
-                new ShutdownAnalyzerTask,
                 $this->taskDoneClosure(...),
             );
 
+            $forked_pool_data = $project_analyzer->pool->runAll(new ShutdownAnalyzerTask);
+
             $this->progress->debug('Collecting forked analysis results' . "\n");
 
             foreach ($forked_pool_data as $pool_data) {

diff --git a/src/Psalm/Internal/Codebase/Scanner.php b/src/Psalm/Internal/Codebase/Scanner.php
@@ -8,14 +8,12 @@
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
 use Psalm\Internal\ErrorHandler;
 use Psalm\Internal\Fork\InitScannerTask;
-use Psalm\Internal\Fork\Pool;
 use Psalm\Internal\Fork\ScannerTask;
 use Psalm\Internal\Fork\ShutdownScannerTask;
 use Psalm\Internal\Provider\FileProvider;
 use Psalm\Internal\Provider\FileReferenceProvider;
 use Psalm\Internal\Provider\FileStorageProvider;
 use Psalm\Internal\Scanner\FileScanner;
-use Psalm\IssueBuffer;
 use Psalm\Progress\Progress;
 use Psalm\Storage\ClassLikeStorage;
 use Psalm\Storage\FileStorage;
@@ -27,12 +25,10 @@
 use function array_filter;
 use function array_merge;
 use function array_pop;
-use function ceil;
 use function count;
 use function error_reporting;
 use function explode;
 use function file_exists;
-use function min;
 use function realpath;
 use function strtolower;
 use function substr;
@@ -314,48 +310,21 @@ private function scanFilePaths(): bool
         $pool_size = $this->is_forked ? 1 : $project_analyzer->threads;
 
         if ($pool_size > 1) {
-            $this->progress->debug('Forking process for scanning' . PHP_EOL);
+            $cnt = count($files_to_scan);
+            $this->progress->debug("Sending {$cnt} files to pool for scanning" . PHP_EOL);
 
-            $forked_pool_data = ProjectAnalyzer::getInstance()->pool->run(
+            $pool = ProjectAnalyzer::getInstance()->pool;
+            $pool->runAll(new InitScannerTask);
+
+            $pool->run(
                 $files_to_scan,
-                new InitScannerTask(),
                 ScannerTask::class,
-                new ShutdownScannerTask,
             );
 
+            $forked_pool_data = $pool->runAll(new ShutdownScannerTask);
             foreach ($forked_pool_data as $pool_data) {
-                IssueBuffer::addIssues($pool_data['issues']);
-
-                $this->codebase->statements_provider->addChangedMembers(
-                    $pool_data['changed_members'],
-                );
-                $this->codebase->statements_provider->addUnchangedSignatureMembers(
-                    $pool_data['unchanged_signature_members'],
-                );
-                $this->codebase->statements_provider->addDiffMap(
-                    $pool_data['diff_map'],
-                );
-                $this->codebase->statements_provider->addDeletionRanges(
-                    $pool_data['deletion_ranges'],
-                );
-                $this->codebase->statements_provider->addErrors($pool_data['errors']);
-
-                if ($this->codebase->taint_flow_graph && $pool_data['taint_data']) {
-                    $this->codebase->taint_flow_graph->addGraph($pool_data['taint_data']);
-                }
-
-                $this->codebase->file_storage_provider->addMore($pool_data['file_storage']);
-                $this->codebase->classlike_storage_provider->addMore($pool_data['classlike_storage']);
-
-                $this->codebase->classlikes->addThreadData($pool_data['classlikes_data']);
-
+                $this->codebase->addThreadData($pool_data);
                 $this->addThreadData($pool_data['scanner_data']);
-
-                if ($this->codebase->statements_provider->parser_cache_provider) {
-                    $this->codebase->statements_provider->parser_cache_provider->addNewFileContentHashes(
-                        $pool_data['new_file_content_hashes'],
-                    );
-                }
             }
         } else {
             $i = 0;

diff --git a/src/Psalm/Internal/Codebase/TaintFlowGraph.php b/src/Psalm/Internal/Codebase/TaintFlowGraph.php
@@ -521,7 +521,7 @@ private function getSpecializedSources(DataFlowNode $source): array
 
         return array_filter(
             $generated_sources,
-            fn ($new_source) => isset($this->forward_edges[$new_source->id])
+            fn($new_source) => isset($this->forward_edges[$new_source->id])
         );
     }
 }
diff --git a/src/Psalm/Internal/Fork/InitAnalyzerTask.php b/src/Psalm/Internal/Fork/InitAnalyzerTask.php
@@ -6,31 +6,20 @@
 use Amp\Parallel\Worker\Task;
 use Amp\Sync\Channel;
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
-use Psalm\Internal\Codebase\TaintFlowGraph;
+use Psalm\Internal\Codebase\Scanner;
 
+/** @psalm-import-type PoolData from Scanner */
 final class InitAnalyzerTask implements Task
 {
+    /** @var PoolData */
+    private readonly array $data;
+    public function __construct()
+    {
+        $this->data = ShutdownScannerTask::getPoolData();
+    }
     public function run(Channel $channel, Cancellation $cancellation): mixed
     {
-        $project_analyzer = ProjectAnalyzer::getInstance();
-        $codebase = $project_analyzer->getCodebase();
-
-        $file_reference_provider = $codebase->file_reference_provider;
-
-        if ($codebase->taint_flow_graph) {
-            $codebase->taint_flow_graph = new TaintFlowGraph();
-        }
-
-        $file_reference_provider->setNonMethodReferencesToClasses([]);
-        $file_reference_provider->setCallingMethodReferencesToClassMembers([]);
-        $file_reference_provider->setCallingMethodReferencesToClassProperties([]);
-        $file_reference_provider->setFileReferencesToClassMembers([]);
-        $file_reference_provider->setFileReferencesToClassProperties([]);
-        $file_reference_provider->setCallingMethodReferencesToMissingClassMembers([]);
-        $file_reference_provider->setFileReferencesToMissingClassMembers([]);
-        $file_reference_provider->setReferencesToMixedMemberNames([]);
-        $file_reference_provider->setMethodParamUses([]);
-
+        ProjectAnalyzer::getInstance()->getCodebase()->addThreadData($this->data);
         return null;
     }
 }
diff --git a/src/Psalm/Internal/Fork/InitScannerTask.php b/src/Psalm/Internal/Fork/InitScannerTask.php
@@ -5,23 +5,9 @@
 use Amp\Cancellation;
 use Amp\Parallel\Worker\Task;
 use Amp\Sync\Channel;
-use Psalm\Config;
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
-use Psalm\Internal\CliUtils;
-use Psalm\Internal\ErrorHandler;
 use Psalm\Internal\Provider\ClassLikeStorageProvider;
 use Psalm\Internal\Provider\FileStorageProvider;
-use Psalm\Internal\VersionUtils;
-use Psalm\IssueBuffer;
-
-use function cli_get_process_title;
-use function cli_set_process_title;
-use function define;
-use function function_exists;
-use function gc_collect_cycles;
-use function gc_disable;
-use function ini_get;
-use function ini_set;
 
 use const PHP_EOL;
 

diff --git a/src/Psalm/Internal/Fork/InitStartupTask.php b/src/Psalm/Internal/Fork/InitStartupTask.php
@@ -9,22 +9,17 @@
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
 use Psalm\Internal\CliUtils;
 use Psalm\Internal\ErrorHandler;
-use Psalm\Internal\Provider\ClassLikeStorageProvider;
-use Psalm\Internal\Provider\FileStorageProvider;
 use Psalm\Internal\VersionUtils;
 use Psalm\IssueBuffer;
 
 use function cli_get_process_title;
-use function cli_set_process_title;
 use function define;
 use function function_exists;
 use function gc_collect_cycles;
 use function gc_disable;
 use function ini_get;
 use function ini_set;
 
-use const PHP_EOL;
-
 final class InitStartupTask implements Task
 {
     private readonly string $memoryLimit;
@@ -60,9 +55,9 @@ final public function run(Channel $channel, Cancellation $cancellation): mixed
         ProjectAnalyzer::$instance = $this->analyzer;
         Config::setInstance($this->analyzer->getConfig());
 
-        if (function_exists('cli_set_process_title') && $this->processTitle !== null) {
+        /*if (function_exists('cli_set_process_title') && $this->processTitle !== null) {
             @cli_set_process_title($this->processTitle);
-        }
+        }*/
 
         return null;
     }

diff --git a/src/Psalm/Internal/Fork/Pool.php b/src/Psalm/Internal/Fork/Pool.php
@@ -12,13 +12,17 @@
 use AssertionError;
 use Closure;
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
+use Psalm\Progress\Progress;
+use Throwable;
 
 use function Amp\Future\await;
 use function array_map;
+use function count;
 use function extension_loaded;
 use function gc_collect_cycles;
 
 use const PHP_BINARY;
+use const PHP_EOL;
 
 /**
  * Adapted with relatively few changes from
@@ -34,6 +38,7 @@
 final class Pool
 {
     private readonly WorkerPool $pool;
+    private readonly Progress $progress;
     /**
      * @param int<2, max> $threads
      */
@@ -57,6 +62,9 @@ public function __construct(private readonly int $threads, ProjectAnalyzer $proj
         );
 
         $this->runAll(new InitStartupTask($project_analyzer));
+        $this->runAll(new InitScannerTask());
+
+        $this->progress = $project_analyzer->progress;
     }
     /**
      * @template TFinalResult
@@ -77,31 +85,38 @@ public function __construct(private readonly int $threads, ProjectAnalyzer $proj
      */
     public function run(
         array $process_task_data_iterator,
-        Task $startup_task,
         string $main_task,
-        Task $shutdown_task,
         ?Closure $task_done_closure = null
-    ): array {
-        $this->runAll($startup_task);
+    ): void {
+        $total = count($process_task_data_iterator);
+        $this->progress->debug("Processing ".$total." tasks...");
+
+        $cnt = 0;
 
         $results = [];
         foreach ($process_task_data_iterator as $file) {
             $results []= $f = $this->pool->submit(new $main_task($file))->getFuture();
             if ($task_done_closure) {
                 $f->map($task_done_closure);
             }
+            $f->catch(fn(Throwable $e) => throw $e);
+            $f->map(function () use (&$cnt, $total): void {
+                $cnt++;
+                if (!($cnt % 10)) {
+                    $percent = (int) (($cnt*100) / $total);
+                    $this->progress->debug("Processing tasks: $cnt/$total ($percent%)...".PHP_EOL);
+                }
+            });
         }
         await($results);
-
-        return $this->runAll($shutdown_task);
     }
 
     /**
      * @template T
      * @param Task<void, void, T> $task
      * @return list<T>
      */
-    private function runAll(Task $task): array
+    public function runAll(Task $task): array
     {
         if ($this->pool->getIdleWorkerCount() !== $this->pool->getWorkerCount()) {
             throw new AssertionError("Some workers are busy!");

diff --git a/src/Psalm/Internal/Fork/ScannerTask.php b/src/Psalm/Internal/Fork/ScannerTask.php
@@ -5,7 +5,6 @@
 use Amp\Cancellation;
 use Amp\Parallel\Worker\Task;
 use Amp\Sync\Channel;
-use AssertionError;
 use Psalm\Internal\Analyzer\ProjectAnalyzer;
 
 /** @internal */