diff --git a/src/Psalm/Codebase.php b/src/Psalm/Codebase.php index 40964b381f9..19d0d02eb91 100644 --- a/src/Psalm/Codebase.php +++ b/src/Psalm/Codebase.php @@ -2,7 +2,6 @@ namespace Psalm; -use Amp\Parallel\Worker\WorkerPool; use Exception; use InvalidArgumentException; use LanguageServerProtocol\Command; @@ -38,7 +37,6 @@ use Psalm\Internal\Codebase\TaintFlowGraph; use Psalm\Internal\DataFlow\TaintSink; use Psalm\Internal\DataFlow\TaintSource; -use Psalm\Internal\Fork\Pool; use Psalm\Internal\LanguageServer\PHPMarkdownContent; use Psalm\Internal\LanguageServer\Reference; use Psalm\Internal\MethodIdentifier; @@ -98,6 +96,7 @@ use const PHP_VERSION_ID; +/** @psalm-import-type PoolData from Scanner */ final class Codebase { /** @@ -394,6 +393,43 @@ private function loadAnalyzer(): void ); } + /** + * @internal + * @param PoolData $pool_data + */ + public function addThreadData(array $pool_data): void + { + IssueBuffer::addIssues($pool_data['issues']); + + $this->statements_provider->addChangedMembers( + $pool_data['changed_members'], + ); + $this->statements_provider->addUnchangedSignatureMembers( + $pool_data['unchanged_signature_members'], + ); + $this->statements_provider->addDiffMap( + $pool_data['diff_map'], + ); + $this->statements_provider->addDeletionRanges( + $pool_data['deletion_ranges'], + ); + $this->statements_provider->addErrors($pool_data['errors']); + + if ($this->taint_flow_graph && $pool_data['taint_data']) { + $this->taint_flow_graph->addGraph($pool_data['taint_data']); + } + + $this->file_storage_provider->addMore($pool_data['file_storage']); + $this->classlike_storage_provider->addMore($pool_data['classlike_storage']); + + $this->classlikes->addThreadData($pool_data['classlikes_data']); + + if ($this->statements_provider->parser_cache_provider) { + $this->statements_provider->parser_cache_provider->addNewFileContentHashes( + $pool_data['new_file_content_hashes'], + ); + } + } /** * @param array $candidate_files */ diff --git a/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php b/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php index 30d3b97b12d..9fdcf80cfa7 100644 --- a/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php +++ b/src/Psalm/Internal/Analyzer/ProjectAnalyzer.php @@ -69,6 +69,7 @@ use function explode; use function file_exists; use function fwrite; +use function get_object_vars; use function implode; use function in_array; use function is_dir; @@ -77,7 +78,6 @@ use function mkdir; use function number_format; use function preg_match; -use function Psl\Type\array_key; use function rename; use function sprintf; use function strlen; diff --git a/src/Psalm/Internal/Codebase/Analyzer.php b/src/Psalm/Internal/Codebase/Analyzer.php index 366deec9379..e381fa94041 100644 --- a/src/Psalm/Internal/Codebase/Analyzer.php +++ b/src/Psalm/Internal/Codebase/Analyzer.php @@ -16,8 +16,6 @@ use Psalm\Internal\FileManipulation\FunctionDocblockManipulator; use Psalm\Internal\FileManipulation\PropertyDocblockManipulator; use Psalm\Internal\Fork\AnalyzerTask; -use Psalm\Internal\Fork\InitAnalyzerTask; -use Psalm\Internal\Fork\Pool; use Psalm\Internal\Fork\ShutdownAnalyzerTask; use Psalm\Internal\Provider\FileProvider; use Psalm\Internal\Provider\FileStorageProvider; @@ -337,14 +335,14 @@ private function doAnalysis(ProjectAnalyzer $project_analyzer): void $this->progress->debug('Forking analysis' . "\n"); - $forked_pool_data = $project_analyzer->pool->run( + $project_analyzer->pool->run( $new_file_paths, - new InitAnalyzerTask(), AnalyzerTask::class, - new ShutdownAnalyzerTask, $this->taskDoneClosure(...), ); + $forked_pool_data = $project_analyzer->pool->runAll(new ShutdownAnalyzerTask); + $this->progress->debug('Collecting forked analysis results' . "\n"); foreach ($forked_pool_data as $pool_data) { diff --git a/src/Psalm/Internal/Codebase/Scanner.php b/src/Psalm/Internal/Codebase/Scanner.php index 8e6675dab80..4cdbd3370a7 100644 --- a/src/Psalm/Internal/Codebase/Scanner.php +++ b/src/Psalm/Internal/Codebase/Scanner.php @@ -8,14 +8,12 @@ use Psalm\Internal\Analyzer\ProjectAnalyzer; use Psalm\Internal\ErrorHandler; use Psalm\Internal\Fork\InitScannerTask; -use Psalm\Internal\Fork\Pool; use Psalm\Internal\Fork\ScannerTask; use Psalm\Internal\Fork\ShutdownScannerTask; use Psalm\Internal\Provider\FileProvider; use Psalm\Internal\Provider\FileReferenceProvider; use Psalm\Internal\Provider\FileStorageProvider; use Psalm\Internal\Scanner\FileScanner; -use Psalm\IssueBuffer; use Psalm\Progress\Progress; use Psalm\Storage\ClassLikeStorage; use Psalm\Storage\FileStorage; @@ -27,12 +25,10 @@ use function array_filter; use function array_merge; use function array_pop; -use function ceil; use function count; use function error_reporting; use function explode; use function file_exists; -use function min; use function realpath; use function strtolower; use function substr; @@ -314,48 +310,21 @@ private function scanFilePaths(): bool $pool_size = $this->is_forked ? 1 : $project_analyzer->threads; if ($pool_size > 1) { - $this->progress->debug('Forking process for scanning' . PHP_EOL); + $cnt = count($files_to_scan); + $this->progress->debug("Sending {$cnt} files to pool for scanning" . PHP_EOL); - $forked_pool_data = ProjectAnalyzer::getInstance()->pool->run( + $pool = ProjectAnalyzer::getInstance()->pool; + $pool->runAll(new InitScannerTask); + + $pool->run( $files_to_scan, - new InitScannerTask(), ScannerTask::class, - new ShutdownScannerTask, ); + $forked_pool_data = $pool->runAll(new ShutdownScannerTask); foreach ($forked_pool_data as $pool_data) { - IssueBuffer::addIssues($pool_data['issues']); - - $this->codebase->statements_provider->addChangedMembers( - $pool_data['changed_members'], - ); - $this->codebase->statements_provider->addUnchangedSignatureMembers( - $pool_data['unchanged_signature_members'], - ); - $this->codebase->statements_provider->addDiffMap( - $pool_data['diff_map'], - ); - $this->codebase->statements_provider->addDeletionRanges( - $pool_data['deletion_ranges'], - ); - $this->codebase->statements_provider->addErrors($pool_data['errors']); - - if ($this->codebase->taint_flow_graph && $pool_data['taint_data']) { - $this->codebase->taint_flow_graph->addGraph($pool_data['taint_data']); - } - - $this->codebase->file_storage_provider->addMore($pool_data['file_storage']); - $this->codebase->classlike_storage_provider->addMore($pool_data['classlike_storage']); - - $this->codebase->classlikes->addThreadData($pool_data['classlikes_data']); - + $this->codebase->addThreadData($pool_data); $this->addThreadData($pool_data['scanner_data']); - - if ($this->codebase->statements_provider->parser_cache_provider) { - $this->codebase->statements_provider->parser_cache_provider->addNewFileContentHashes( - $pool_data['new_file_content_hashes'], - ); - } } } else { $i = 0; diff --git a/src/Psalm/Internal/Codebase/TaintFlowGraph.php b/src/Psalm/Internal/Codebase/TaintFlowGraph.php index 7c5ed519bfb..ce034821db1 100644 --- a/src/Psalm/Internal/Codebase/TaintFlowGraph.php +++ b/src/Psalm/Internal/Codebase/TaintFlowGraph.php @@ -521,7 +521,7 @@ private function getSpecializedSources(DataFlowNode $source): array return array_filter( $generated_sources, - fn ($new_source) => isset($this->forward_edges[$new_source->id]) + fn($new_source) => isset($this->forward_edges[$new_source->id]) ); } } diff --git a/src/Psalm/Internal/Fork/InitAnalyzerTask.php b/src/Psalm/Internal/Fork/InitAnalyzerTask.php index e68ad380ff6..0eeef2e7c76 100644 --- a/src/Psalm/Internal/Fork/InitAnalyzerTask.php +++ b/src/Psalm/Internal/Fork/InitAnalyzerTask.php @@ -6,31 +6,20 @@ use Amp\Parallel\Worker\Task; use Amp\Sync\Channel; use Psalm\Internal\Analyzer\ProjectAnalyzer; -use Psalm\Internal\Codebase\TaintFlowGraph; +use Psalm\Internal\Codebase\Scanner; +/** @psalm-import-type PoolData from Scanner */ final class InitAnalyzerTask implements Task { + /** @var PoolData */ + private readonly array $data; + public function __construct() + { + $this->data = ShutdownScannerTask::getPoolData(); + } public function run(Channel $channel, Cancellation $cancellation): mixed { - $project_analyzer = ProjectAnalyzer::getInstance(); - $codebase = $project_analyzer->getCodebase(); - - $file_reference_provider = $codebase->file_reference_provider; - - if ($codebase->taint_flow_graph) { - $codebase->taint_flow_graph = new TaintFlowGraph(); - } - - $file_reference_provider->setNonMethodReferencesToClasses([]); - $file_reference_provider->setCallingMethodReferencesToClassMembers([]); - $file_reference_provider->setCallingMethodReferencesToClassProperties([]); - $file_reference_provider->setFileReferencesToClassMembers([]); - $file_reference_provider->setFileReferencesToClassProperties([]); - $file_reference_provider->setCallingMethodReferencesToMissingClassMembers([]); - $file_reference_provider->setFileReferencesToMissingClassMembers([]); - $file_reference_provider->setReferencesToMixedMemberNames([]); - $file_reference_provider->setMethodParamUses([]); - + ProjectAnalyzer::getInstance()->getCodebase()->addThreadData($this->data); return null; } } diff --git a/src/Psalm/Internal/Fork/InitScannerTask.php b/src/Psalm/Internal/Fork/InitScannerTask.php index 6330c01ac82..99b1720604e 100644 --- a/src/Psalm/Internal/Fork/InitScannerTask.php +++ b/src/Psalm/Internal/Fork/InitScannerTask.php @@ -5,23 +5,9 @@ use Amp\Cancellation; use Amp\Parallel\Worker\Task; use Amp\Sync\Channel; -use Psalm\Config; use Psalm\Internal\Analyzer\ProjectAnalyzer; -use Psalm\Internal\CliUtils; -use Psalm\Internal\ErrorHandler; use Psalm\Internal\Provider\ClassLikeStorageProvider; use Psalm\Internal\Provider\FileStorageProvider; -use Psalm\Internal\VersionUtils; -use Psalm\IssueBuffer; - -use function cli_get_process_title; -use function cli_set_process_title; -use function define; -use function function_exists; -use function gc_collect_cycles; -use function gc_disable; -use function ini_get; -use function ini_set; use const PHP_EOL; diff --git a/src/Psalm/Internal/Fork/InitStartupTask.php b/src/Psalm/Internal/Fork/InitStartupTask.php index 4f95673beb8..3f567524bdf 100644 --- a/src/Psalm/Internal/Fork/InitStartupTask.php +++ b/src/Psalm/Internal/Fork/InitStartupTask.php @@ -9,13 +9,10 @@ use Psalm\Internal\Analyzer\ProjectAnalyzer; use Psalm\Internal\CliUtils; use Psalm\Internal\ErrorHandler; -use Psalm\Internal\Provider\ClassLikeStorageProvider; -use Psalm\Internal\Provider\FileStorageProvider; use Psalm\Internal\VersionUtils; use Psalm\IssueBuffer; use function cli_get_process_title; -use function cli_set_process_title; use function define; use function function_exists; use function gc_collect_cycles; @@ -23,8 +20,6 @@ use function ini_get; use function ini_set; -use const PHP_EOL; - final class InitStartupTask implements Task { private readonly string $memoryLimit; @@ -60,9 +55,9 @@ final public function run(Channel $channel, Cancellation $cancellation): mixed ProjectAnalyzer::$instance = $this->analyzer; Config::setInstance($this->analyzer->getConfig()); - if (function_exists('cli_set_process_title') && $this->processTitle !== null) { + /*if (function_exists('cli_set_process_title') && $this->processTitle !== null) { @cli_set_process_title($this->processTitle); - } + }*/ return null; } diff --git a/src/Psalm/Internal/Fork/Pool.php b/src/Psalm/Internal/Fork/Pool.php index 277a7e12713..c7b29d4ca8c 100644 --- a/src/Psalm/Internal/Fork/Pool.php +++ b/src/Psalm/Internal/Fork/Pool.php @@ -12,13 +12,17 @@ use AssertionError; use Closure; use Psalm\Internal\Analyzer\ProjectAnalyzer; +use Psalm\Progress\Progress; +use Throwable; use function Amp\Future\await; use function array_map; +use function count; use function extension_loaded; use function gc_collect_cycles; use const PHP_BINARY; +use const PHP_EOL; /** * Adapted with relatively few changes from @@ -34,6 +38,7 @@ final class Pool { private readonly WorkerPool $pool; + private readonly Progress $progress; /** * @param int<2, max> $threads */ @@ -57,6 +62,9 @@ public function __construct(private readonly int $threads, ProjectAnalyzer $proj ); $this->runAll(new InitStartupTask($project_analyzer)); + $this->runAll(new InitScannerTask()); + + $this->progress = $project_analyzer->progress; } /** * @template TFinalResult @@ -77,12 +85,13 @@ public function __construct(private readonly int $threads, ProjectAnalyzer $proj */ public function run( array $process_task_data_iterator, - Task $startup_task, string $main_task, - Task $shutdown_task, ?Closure $task_done_closure = null - ): array { - $this->runAll($startup_task); + ): void { + $total = count($process_task_data_iterator); + $this->progress->debug("Processing ".$total." tasks..."); + + $cnt = 0; $results = []; foreach ($process_task_data_iterator as $file) { @@ -90,10 +99,16 @@ public function run( if ($task_done_closure) { $f->map($task_done_closure); } + $f->catch(fn(Throwable $e) => throw $e); + $f->map(function () use (&$cnt, $total): void { + $cnt++; + if (!($cnt % 10)) { + $percent = (int) (($cnt*100) / $total); + $this->progress->debug("Processing tasks: $cnt/$total ($percent%)...".PHP_EOL); + } + }); } await($results); - - return $this->runAll($shutdown_task); } /** @@ -101,7 +116,7 @@ public function run( * @param Task $task * @return list */ - private function runAll(Task $task): array + public function runAll(Task $task): array { if ($this->pool->getIdleWorkerCount() !== $this->pool->getWorkerCount()) { throw new AssertionError("Some workers are busy!"); diff --git a/src/Psalm/Internal/Fork/ScannerTask.php b/src/Psalm/Internal/Fork/ScannerTask.php index 8f2c256390e..ab9764ba0ca 100644 --- a/src/Psalm/Internal/Fork/ScannerTask.php +++ b/src/Psalm/Internal/Fork/ScannerTask.php @@ -5,7 +5,6 @@ use Amp\Cancellation; use Amp\Parallel\Worker\Task; use Amp\Sync\Channel; -use AssertionError; use Psalm\Internal\Analyzer\ProjectAnalyzer; /** @internal */ diff --git a/src/Psalm/Internal/Fork/ShutdownScannerTask.php b/src/Psalm/Internal/Fork/ShutdownScannerTask.php index 0fb88ba0819..3276036cbe6 100644 --- a/src/Psalm/Internal/Fork/ShutdownScannerTask.php +++ b/src/Psalm/Internal/Fork/ShutdownScannerTask.php @@ -22,6 +22,13 @@ final class ShutdownScannerTask implements Task * @return PoolData */ public function run(Channel $channel, Cancellation $cancellation): mixed + { + return self::getPoolData(); + } + /** + * @return PoolData + */ + public static function getPoolData(): array { $project_analyzer = ProjectAnalyzer::getInstance(); $project_analyzer->progress->debug('Collecting data from forked scanner process' . PHP_EOL); diff --git a/src/Psalm/Internal/Provider/ClassLikeStorageProvider.php b/src/Psalm/Internal/Provider/ClassLikeStorageProvider.php index 9b8c82617ea..3d988275674 100644 --- a/src/Psalm/Internal/Provider/ClassLikeStorageProvider.php +++ b/src/Psalm/Internal/Provider/ClassLikeStorageProvider.php @@ -83,7 +83,7 @@ public function exhume(string $fq_classlike_name, string $file_path, string $fil /** * @return array */ - public function getAll(): array + public static function getAll(): array { return self::$storage; } diff --git a/src/Psalm/Internal/Provider/FileStorageProvider.php b/src/Psalm/Internal/Provider/FileStorageProvider.php index 946d3c46062..b686b397c08 100644 --- a/src/Psalm/Internal/Provider/FileStorageProvider.php +++ b/src/Psalm/Internal/Provider/FileStorageProvider.php @@ -83,7 +83,7 @@ public function has(string $file_path, ?string $file_contents = null): bool /** * @return array */ - public function getAll(): array + public static function getAll(): array { return self::$storage; }