diff --git a/cleantalk.php b/cleantalk.php index f5e142af6..0316db41e 100644 --- a/cleantalk.php +++ b/cleantalk.php @@ -1531,92 +1531,13 @@ function apbct_sfw_update__get_multifiles_of_type(array $params) /** * Queue stage. Do load multifiles with networks on their urls. - * @param $urls - * @return array|array[]|bool|string|string[] + * @param $all_urls + * @return array|true */ -function apbct_sfw_update__download_files($urls, $direct_update = false) +function apbct_sfw_update__download_files($all_urls, $direct_update = false) { - global $apbct; - - sleep(3); - - if ( ! is_writable($apbct->fw_stats['updating_folder']) ) { - return array('error' => 'SFW update folder is not writable.'); - } - - //Reset keys - $urls = array_values(array_unique($urls)); - - $results = array(); - $batch_size = 10; - - /** - * Reduce batch size of curl multi instanced - */ - if (defined('APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE')) { - if ( - is_int(APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE) && - APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE > 0 && - APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE < 10 - ) { - $batch_size = APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE; - }; - } - - $total_urls = count($urls); - $batches = ceil($total_urls / $batch_size); - - for ($i = 0; $i < $batches; $i++) { - $batch_urls = array_slice($urls, $i * $batch_size, $batch_size); - if (!empty($batch_urls)) { - $http_results = Helper::httpMultiRequest($batch_urls, $apbct->fw_stats['updating_folder']); - if (is_array($http_results)) { - $results = array_merge($results, $http_results); - } - // to handle case if we request only one url, then Helper::httpMultiRequest returns string 'success' instead of array - if (count($batch_urls) === 1 && $http_results === 'success') { - $results = array_merge($results, $batch_urls); - } - } - } - - $results = TT::toArray($results); - $count_urls = count($urls); - $count_results = count($results); - - if ( empty($results['error']) && ($count_urls === $count_results) ) { - if ( $direct_update ) { - return true; - } - $download_again = array(); - $results = array_values($results); - for ( $i = 0; $i < $count_results; $i++ ) { - if ( $results[$i] === 'error' ) { - $download_again[] = $urls[$i]; - } - } - - if ( count($download_again) !== 0 ) { - return array( - 'error' => 'Files download not completed.', - 'update_args' => array( - 'args' => $download_again - ) - ); - } - - return array( - 'next_stage' => array( - 'name' => 'apbct_sfw_update__create_tables' - ) - ); - } - - if ( ! empty($results['error']) ) { - return $results; - } - - return array('error' => 'Files download not completed.'); + $downloader = new \Cleantalk\ApbctWP\Firewall\SFWFilesDownloader(); + return $downloader->downloadFiles($all_urls, $direct_update); } /** diff --git a/lib/Cleantalk/ApbctWP/Firewall/SFWFilesDownloader.php b/lib/Cleantalk/ApbctWP/Firewall/SFWFilesDownloader.php new file mode 100644 index 000000000..40f6f7d82 --- /dev/null +++ b/lib/Cleantalk/ApbctWP/Firewall/SFWFilesDownloader.php @@ -0,0 +1,205 @@ +deafult_error_prefix = basename(__CLASS__) . ': '; + $this->http_multi_request_factory = $factory ?: new HTTPMultiRequestFactory(); + } + + /** + * Downloads SFW data files from provided URLs with batch processing and retry logic + * + * Downloads files in batches to avoid server overload. Automatically retries failed downloads + * with reduced batch size if necessary. Validates write permissions and URL format before processing. + * + * @param array|mixed $all_urls List of URLs to download files from + * @param bool $direct_update Optional. If true, returns boolean result. If false, returns stage info array. + * @param int sleep Pause in seconds before multi contracts run, default is 3 + * + * @return true|array True on success (direct update mode), or array with 'next_stage' key, + * or array with 'error' key on failure, or array with 'update_args' for retry. + */ + public function downloadFiles($all_urls, $direct_update = false, $sleep = 3) + { + global $apbct; + + // Delay to prevent server overload + sleep($sleep); + + // Validate write permissions for update folder + if ( ! is_writable($apbct->fw_stats['updating_folder']) ) { + return $this->responseStopUpdate('SFW UPDATE FOLDER IS NOT WRITABLE.'); + } + + // Validate URLs parameter type + if ( ! is_array($all_urls) ) { + return $this->responseStopUpdate('URLS LIST SHOULD BE AN ARRAY'); + } + + // Remove duplicates and reset array keys to sequential integers + $all_urls = array_values(array_unique($all_urls)); + + // Get current batch size from settings or default + $work_batch_size = SFWUpdateHelper::getSFWFilesBatchSize(); + + // Initialize batch processing variables + $total_urls = count($all_urls); + $batches = ceil($total_urls / $work_batch_size); + $download_again = []; + $written_urls = []; + + // Get or set default batch size for retry attempts + $on_repeat_batch_size = !empty($apbct->data['sfw_update__batch_size']) + ? TT::toInt($apbct->data['sfw_update__batch_size']) + : 10; + + // Process URLs in batches + for ($i = 0; $i < $batches; $i++) { + // Extract current batch of URLs + $current_batch_urls = array_slice($all_urls, $i * $work_batch_size, $work_batch_size); + + if (!empty($current_batch_urls)) { + // Execute multi-request for current batch + $multi_request_contract = $this->http_multi_request_factory->setMultiContract($current_batch_urls); + + // Critical error: contract processing failed, stop update immediately + if (!$multi_request_contract->process_done) { + $error = !empty($multi_request_contract->error_msg) ? $multi_request_contract->error_msg : 'UNKNOWN ERROR'; + return $this->responseStopUpdate($error); + } + + // Handle failed downloads in this batch + if (!empty($multi_request_contract->getFailedURLs())) { + // Reduce batch size for retry if fabric suggests it + if ($multi_request_contract->suggest_batch_reduce_to) { + $on_repeat_batch_size = min($on_repeat_batch_size, $multi_request_contract->suggest_batch_reduce_to); + } + // Collect failed URLs for retry + $download_again = array_merge($download_again, $multi_request_contract->getFailedURLs()); + } + + // Write successfully downloaded content to files + $write_result = $multi_request_contract->writeSuccessURLsContent($apbct->fw_stats['updating_folder']); + + // File write error occurred, stop update + if (is_string($write_result)) { + return $this->responseStopUpdate($write_result); + } + + // Track successfully written URLs + $written_urls = array_merge($written_urls, $write_result); + } + } + + // Some downloads failed, schedule retry with adjusted batch size + if (!empty($download_again)) { + $apbct->fw_stats['multi_request_batch_size'] = $on_repeat_batch_size; + $apbct->save('data'); + return $this->responseRepeatStage('FILES DOWNLOAD NOT COMPLETED, TRYING AGAIN', $download_again); + } + + // Verify all URLs were successfully downloaded and written + if (empty(array_diff($all_urls, $written_urls))) { + return $this->responseSuccess($direct_update); + } + + // Download incomplete with no retry - collect error information + $last_contract_errors = isset($multi_request_contract) && $multi_request_contract->getContractsErrors() + ? $multi_request_contract->getContractsErrors() + : 'no known contract errors'; + + $error = 'FILES DOWNLOAD NOT COMPLETED - STOP UPDATE, ERRORS: ' . $last_contract_errors; + return $this->responseStopUpdate($error); + } + + /** + * Creates error response to stop the update process + * + * @param string $message Error message describing why update was stopped + * + * @return array Error response array with 'error' key + */ + private function responseStopUpdate($message): array + { + $message = is_string($message) ? $message : $this->deafult_error_content; + $message = $this->deafult_error_prefix . $message; + return [ + 'error' => $message + ]; + } + + /** + * Creates response to repeat current stage with modified arguments + * + * Used when downloads partially failed and should be retried with + * potentially reduced batch size or different parameters. + * + * @param string $message Descriptive message about why stage needs repeating + * @param array $args Arguments for retry attempt (typically failed URLs) + * + * @return array Response array with 'error' message and 'update_args' for retry + */ + private function responseRepeatStage($message, $args): array + { + $args = is_array($args) ? $args : []; + $message = is_string($message) ? $message : $this->deafult_error_content; + $message = $this->deafult_error_prefix . $message; + return [ + 'error' => $message, + 'update_args' => [ + 'args' => $args + ], + ]; + } + + /** + * Creates success response to proceed to next stage or complete update + * + * @param bool $direct_update If true, returns simple boolean. If false, returns stage transition array. + * + * @return true|array True for direct update mode, or array with 'next_stage' key for staged updates + */ + private function responseSuccess($direct_update) + { + return $direct_update ? true : [ + 'next_stage' => array( + 'name' => 'apbct_sfw_update__create_tables' + ) + ]; + } +} diff --git a/lib/Cleantalk/ApbctWP/Firewall/SFWUpdateHelper.php b/lib/Cleantalk/ApbctWP/Firewall/SFWUpdateHelper.php index 70039a8f3..4dd7697fd 100644 --- a/lib/Cleantalk/ApbctWP/Firewall/SFWUpdateHelper.php +++ b/lib/Cleantalk/ApbctWP/Firewall/SFWUpdateHelper.php @@ -578,4 +578,29 @@ public static function fallback() */ apbct_sfw_update__create_tables(); } + + public static function getSFWFilesBatchSize() + { + global $apbct; + $work_batch_size = 10; + + /** + * Reduce batch size of curl multi instanced + */ + if (defined('APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE')) { + if ( + is_int(APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE) && + APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE > 1 && + APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE < 10 + ) { + $work_batch_size = APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE; + }; + } + + $work_batch_size = !empty($apbct->fw_stats['multi_request_batch_size']) + ? TT::toInt($apbct->fw_stats['multi_request_batch_size'], $work_batch_size) + : $work_batch_size; + + return $work_batch_size; + } } diff --git a/lib/Cleantalk/ApbctWP/HTTP/HTTPMultiRequestFactory.php b/lib/Cleantalk/ApbctWP/HTTP/HTTPMultiRequestFactory.php new file mode 100644 index 000000000..20f4473b5 --- /dev/null +++ b/lib/Cleantalk/ApbctWP/HTTP/HTTPMultiRequestFactory.php @@ -0,0 +1,360 @@ +process_done = false; + $this->suggest_batch_reduce_to = false; + $this->contracts = []; + $this->error_msg = null; + + // Prepare individual contracts for each URL + $this->prepareContracts($urls); + + // Execute all HTTP requests if contracts are valid + $this->executeMultiContract(); + + return $this; + } + + /** + * Prepares HTTP request contracts from URLs array + * + * Validates URLs and creates HTTPRequestContract instance for each valid URL. + * Sets error message and stops processing if validation fails. + * + * @param array $urls Array of URLs to prepare contracts for + * + * @return void + */ + public function prepareContracts(array $urls) + { + // Validate URLs array is not empty + if (empty($urls)) { + $this->error_msg = __CLASS__ . ': URLS SHOULD BE NOT EMPTY'; + return; + } + + // Create contract for each URL with validation + foreach ($urls as $url) { + // Ensure each URL is a string + if (!is_string($url)) { + $this->error_msg = __CLASS__ . ': SINGLE URL SHOULD BE A STRING'; + $this->contracts = []; + return; + } + $this->contracts[] = new HTTPRequestContract($url); + } + } + + /** + * Executes multi-request and fills contracts with response data + * + * Sends HTTP requests for all prepared contracts and processes the results. + * Only executes if there are valid URLs to process. + * + * @return void + */ + public function executeMultiContract() + { + // Execute requests only if contracts contain URLs + if (!empty($this->getAllURLs())) { + $http_multi_result = $this->sendRequests($this->getAllURLs()); + $this->fillMultiContract($http_multi_result); + } + } + + /** + * Fills contracts with HTTP response data and validates results + * + * Processes multi-request results, validates response content, updates contract states, + * and suggests batch size reduction if some requests failed. + * + * @param array|bool $http_multi_result Response data from multi-request or false on failure + * + * @return $this Returns self for method chaining + */ + public function fillMultiContract($http_multi_result) + { + // Handle HTTP request error + if (!empty($http_multi_result['error'])) { + $this->error_msg = __CLASS__ . ': HTTP_MULTI_RESULT ERROR' . $http_multi_result['error']; + return $this; + } + + // Validate result is an array + if (!is_array($http_multi_result)) { + $this->error_msg = __CLASS__ . ': HTTP_MULTI_RESULT INVALID'; + return $this; + } + + // Fill each contract with corresponding response data + foreach ($this->contracts as $contract) { + if (isset($http_multi_result[$contract->url])) { + $contract_content = $http_multi_result[$contract->url]; + + // Validate response content is string + if (!is_string($contract_content)) { + $contract->error_msg = __CLASS__ . ': SINGLE CONTRACT_CONTENT SHOULD BE A STRING'; + continue; + } + + // Validate response content is not empty + if (empty($contract_content)) { + $contract->error_msg = __CLASS__ . ': SINGLE CONTRACT_CONTENT SHOULD BE NOT EMPTY'; + continue; + } + + // Mark contract as successful with content + $contract->content = $contract_content; + $contract->success = true; + } + } + + // Suggest batch size reduction if some contracts failed + if (!$this->allContractsCompleted()) { + // Reduce to number of successful requests, minimum 2 + $this->suggest_batch_reduce_to = !empty($this->getSuccessURLs()) + ? count($this->getSuccessURLs()) + : 2; + } + + // Mark processing as completed + $this->process_done = true; + return $this; + } + + /** + * Checks if all contracts completed successfully + * + * @return bool True if all contracts succeeded, false if any failed + */ + private function allContractsCompleted() + { + // Extract success flags from all contracts + $flags = array_map(function ($contract) { + return $contract->success; + }, $this->contracts); + + // Return true only if no false flags exist + return !in_array(false, $flags, true); + } + + /** + * Sends HTTP requests to multiple URLs using CommonRequest + * + * @param array $urls Array of URLs to request + * + * @return array|bool Response data array or false on failure + */ + public function sendRequests($urls) + { + $http = new CommonRequest(); + + // Configure and execute multi-request + $http->setUrl($urls) + ->setPresets('get'); + return $http->request(); + } + + /** + * Collects and formats error messages from failed contracts + * + * Returns formatted string with URL and error message pairs for all failed contracts. + * Used for debugging and error reporting. + * + * @return false|string False if no errors, comma-separated error string otherwise + * @psalm-suppress PossiblyUnusedMethod + */ + public function getContractsErrors() + { + $result = []; + + // Collect errors from failed contracts + foreach ($this->contracts as $contract) { + if (!$contract->success && !empty($contract->error_msg)) { + // Format: [url]:[error_message] + $result[] = '[' . esc_url($contract->url) . ']:[' . esc_html($contract->error_msg) . ']'; + } + } + + return empty($result) ? false : implode(',', $result); + } + + /** + * Extracts URLs from all contracts + * + * @return array Array of URLs from all contracts + */ + public function getAllURLs() + { + return array_map(function ($contract) { + return $contract->url; + }, $this->contracts); + } + + /** + * Returns URLs of failed contracts + * + * Contract is considered failed if success flag is false or content is empty. + * + * @return array Array of URLs that failed to download or have empty content + */ + public function getFailedURLs() + { + $result = []; + foreach ($this->contracts as $contract) { + // Failed if not successful or content is empty + if (!$contract->success || empty($contract->content)) { + $result[] = $contract->url; + } + } + return $result; + } + + /** + * Returns URLs of successful contracts + * + * Contract is considered successful if success flag is true and content is not empty. + * + * @return array Array of URLs that successfully downloaded with non-empty content + */ + public function getSuccessURLs() + { + $result = []; + foreach ($this->contracts as $contract) { + // Successful only if both success flag and content exist + if ($contract->success && !empty($contract->content)) { + $result[] = $contract->url; + } + } + return $result; + } + + /** + * Writes content from successful contracts to files + * + * Extracts filename from URL and writes contract content to specified directory. + * Only processes contracts that completed successfully. Validates directory permissions + * before writing and handles write errors gracefully. + * + * @param string $write_to_dir Target directory path (must exist and be writable) + * + * @return array|string Array of successfully written URLs on success, error message string on failure + */ + public function writeSuccessURLsContent($write_to_dir) + { + $written_urls = []; + try { + // Validate target directory exists and is writable + if (!is_dir($write_to_dir) || !is_writable($write_to_dir)) { + throw new \Exception('CAN NOT WRITE TO DIRECTORY: ' . $write_to_dir); + } + + // Write content from each successful contract + foreach ($this->contracts as $single_contract) { + if ($single_contract->success) { + // Extract filename from URL and build full path + $file_name = $write_to_dir . self::getFilenameFromUrl($single_contract->url); + + // Write content to file + $write_result = $this->writeFile($file_name, $single_contract->content); + + // Check for write failure + if (false === $write_result) { + throw new \Exception('CAN NOT WRITE TO FILE: ' . $file_name); + } + + // Track successfully written URL + $written_urls[] = $single_contract->url; + } + } + } catch (\Exception $e) { + // Return error message on any exception + return $e->getMessage(); + } + + return $written_urls; + } + + /** + * Extracts filename with extension from URL + * + * Parses URL to extract filename and extension components. + * Example: "https://example.com/path/file.gz" -> "file.gz" + * + * @param string $url Full URL to extract filename from + * + * @return string Filename with extension + */ + private static function getFilenameFromUrl($url) + { + return pathinfo($url, PATHINFO_FILENAME) . '.' . pathinfo($url, PATHINFO_EXTENSION); + } + + /** + * Wrapper for file_put_contents. + * @param $filename + * @param $data + * @return false|int + */ + public function writeFile($filename, $data) + { + return @file_put_contents($filename, $data); + } +} diff --git a/lib/Cleantalk/ApbctWP/HTTP/HTTPRequestContract.php b/lib/Cleantalk/ApbctWP/HTTP/HTTPRequestContract.php new file mode 100644 index 000000000..f7a8da99b --- /dev/null +++ b/lib/Cleantalk/ApbctWP/HTTP/HTTPRequestContract.php @@ -0,0 +1,28 @@ +url = $url; + } +} diff --git a/lib/Cleantalk/ApbctWP/State.php b/lib/Cleantalk/ApbctWP/State.php index 065e72bfa..de16bc057 100644 --- a/lib/Cleantalk/ApbctWP/State.php +++ b/lib/Cleantalk/ApbctWP/State.php @@ -369,6 +369,7 @@ class State extends \Cleantalk\Common\State 'expected_ua_count_personal' => 0, 'update_mode' => 0, 'reason_direct_update_log' => null, + 'multi_request_batch_size' => 10, 'personal_lists_url_id' => '', 'common_lists_url_id' => '', 'calls' => 0, diff --git a/lib/Cleantalk/Common/Helper.php b/lib/Cleantalk/Common/Helper.php index 66010cc3d..78beb40e1 100644 --- a/lib/Cleantalk/Common/Helper.php +++ b/lib/Cleantalk/Common/Helper.php @@ -614,6 +614,7 @@ public static function httpRequest($url, $data = array(), $presets = array(), $o * @param array $urls Array of URLs to requests * * @return array|bool + * @psalm-suppress PossiblyUnusedMethod */ public static function httpMultiRequest($urls, $write_to = '') { diff --git a/tests/ApbctWP/Firewall/SFWUpdateHelperTest.php b/tests/ApbctWP/Firewall/SFWUpdateHelperTest.php new file mode 100644 index 000000000..ff8454759 --- /dev/null +++ b/tests/ApbctWP/Firewall/SFWUpdateHelperTest.php @@ -0,0 +1,86 @@ +apbctBackup = $apbct; + + // Setup mock $apbct global + $apbct = new \stdClass(); + $apbct->fw_stats['multi_request_batch_size'] = 10; + } + + protected function tearDown() + { + parent::tearDown(); + global $apbct; + $apbct = $this->apbctBackup; + } + + /** + * @test + */ + public function testGetSFWFilesBatchSizeReturnsDefaultValue() + { + global $apbct; + unset($apbct->fw_stats['multi_request_batch_size']); + + $batchSize = SFWUpdateHelper::getSFWFilesBatchSize(); + + $this->assertEquals(10, $batchSize); + } + + /** + * @test + */ + public function testGetSFWFilesBatchSizeReturnsCustomValue() + { + global $apbct; + $apbct->fw_stats['multi_request_batch_size'] = 5; + + $batchSize = SFWUpdateHelper::getSFWFilesBatchSize(); + + $this->assertEquals(5, $batchSize); + } + + /** + * @test + */ + public function testGetSFWFilesBatchSizeReturnsCustomValueAsString() + { + global $apbct; + $apbct->fw_stats['multi_request_batch_size'] = 7; + + $batchSize = SFWUpdateHelper::getSFWFilesBatchSize(); + + $this->assertEquals(7, $batchSize); + } + + /** + * @test + */ + public function testGetSFWFilesBatchSizeRespectsConstantWithValidValue() + { + if (!defined('APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE')) { + define('APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE', 3); + } + + global $apbct; + unset($apbct->fw_stats['multi_request_batch_size']); + + $batchSize = SFWUpdateHelper::getSFWFilesBatchSize(); + + $this->assertEquals(3, $batchSize); + } +} + diff --git a/tests/ApbctWP/HTTP/TestHTTPMultiRequestFactory.php b/tests/ApbctWP/HTTP/TestHTTPMultiRequestFactory.php new file mode 100644 index 000000000..a5a298500 --- /dev/null +++ b/tests/ApbctWP/HTTP/TestHTTPMultiRequestFactory.php @@ -0,0 +1,562 @@ +testFolder = sys_get_temp_dir() . '/test_fabric_' . time() . '/'; + if (!is_dir($this->testFolder)) { + mkdir($this->testFolder, 0777, true); + } + } + + protected function tearDown() + { + parent::tearDown(); + + if (is_dir($this->testFolder)) { + $files = glob($this->testFolder . '/*'); + if ($files) { + foreach ($files as $file) { + if (is_file($file)) { + unlink($file); + } + } + } + rmdir($this->testFolder); + } + } + + /** + * @test + */ + public function testPrepareContractsWithEmptyUrlsSetsError() + { + $fabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->setMethods(['executeMultiContract', 'sendMultiRequest']) + ->getMock(); + + $fabric->expects($this->never()) + ->method('sendMultiRequest'); + + $fabric->setMultiContract([]); + + $this->assertNotNull($fabric->error_msg); + $this->assertStringContainsString('URLS SHOULD BE NOT EMPTY', $fabric->error_msg); + $this->assertEmpty($fabric->contracts); + } + + /** + * @test + */ + public function testPrepareContractsWithNonStringUrlSetsError() + { + $fabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->setMethods(['executeMultiContract']) + ->getMock(); + + $urls = [ + 'https://example.com/file1.gz', + 123, // Invalid + 'https://example.com/file3.gz' + ]; + + $fabric->setMultiContract($urls); + + $this->assertNotNull($fabric->error_msg); + $this->assertStringContainsString('SINGLE URL SHOULD BE A STRING', $fabric->error_msg); + $this->assertEmpty($fabric->contracts); + } + + /** + * @test + */ + public function testPrepareContractsCreatesHTTPRequestContracts() + { + $fabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->setMethods(['executeMultiContract']) + ->getMock(); + + $urls = [ + 'https://example.com/file1.gz', + 'https://example.com/file2.gz' + ]; + + $fabric->setMultiContract($urls); + + $this->assertCount(2, $fabric->contracts); + $this->assertContainsOnlyInstancesOf(HTTPRequestContract::class, $fabric->contracts); + $this->assertEquals($urls[0], $fabric->contracts[0]->url); + $this->assertEquals($urls[1], $fabric->contracts[1]->url); + } + + /** + * @test + */ + public function testGetAllURLsReturnsAllContractUrls() + { + $fabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->setMethods(['executeMultiContract']) + ->getMock(); + + $urls = [ + 'https://example.com/file1.gz', + 'https://example.com/file2.gz' + ]; + + $fabric->setMultiContract($urls); + + $this->assertEquals($urls, $fabric->getAllURLs()); + } + + /** + * @test + */ + public function testGetFailedURLsReturnsUrlsWithNoSuccess() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content'; + + $contract2 = new HTTPRequestContract('https://example.com/file2.gz'); + $contract2->success = false; + + $contract3 = new HTTPRequestContract('https://example.com/file3.gz'); + $contract3->success = true; + $contract3->content = ''; + + $fabric->contracts = [$contract1, $contract2, $contract3]; + + $failed = $fabric->getFailedURLs(); + + $this->assertCount(2, $failed); + $this->assertContains('https://example.com/file2.gz', $failed); + $this->assertContains('https://example.com/file3.gz', $failed); + } + + /** + * @test + */ + public function testGetSuccessURLsReturnsOnlySuccessfulUrls() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content1'; + + $contract2 = new HTTPRequestContract('https://example.com/file2.gz'); + $contract2->success = false; + + $contract3 = new HTTPRequestContract('https://example.com/file3.gz'); + $contract3->success = true; + $contract3->content = 'content3'; + + $fabric->contracts = [$contract1, $contract2, $contract3]; + + $success = $fabric->getSuccessURLs(); + + $this->assertCount(2, $success); + $this->assertEquals(['https://example.com/file1.gz', 'https://example.com/file3.gz'], $success); + } + + /** + * @test + */ + public function testFillMultiContractWithErrorArraySetsError() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->fillMultiContract(['error' => 'CURL_ERROR']); + + $this->assertNotNull($fabric->error_msg); + $this->assertStringContainsString('HTTP_MULTI_RESULT ERROR', $fabric->error_msg); + } + + /** + * @test + */ + public function testFillMultiContractWithNonArraySetsError() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->fillMultiContract('not an array'); + + $this->assertNotNull($fabric->error_msg); + $this->assertStringContainsString('HTTP_MULTI_RESULT INVALID', $fabric->error_msg); + } + + /** + * @test + */ + public function testFillMultiContractWithValidResultsFillsContracts() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->contracts = [ + new HTTPRequestContract('https://example.com/file1.gz'), + new HTTPRequestContract('https://example.com/file2.gz') + ]; + + $results = [ + 'https://example.com/file1.gz' => 'content for file1', + 'https://example.com/file2.gz' => 'content for file2' + ]; + + $fabric->fillMultiContract($results); + + $this->assertTrue($fabric->contracts[0]->success); + $this->assertEquals('content for file1', $fabric->contracts[0]->content); + $this->assertTrue($fabric->contracts[1]->success); + $this->assertEquals('content for file2', $fabric->contracts[1]->content); + $this->assertTrue($fabric->process_done); + } + + /** + * @test + */ + public function testFillMultiContractWithNonStringContentSetsContractError() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->contracts = [ + new HTTPRequestContract('https://example.com/file1.gz') + ]; + + $results = [ + 'https://example.com/file1.gz' => 123 + ]; + + $fabric->fillMultiContract($results); + + $this->assertFalse($fabric->contracts[0]->success); + $this->assertNotNull($fabric->contracts[0]->error_msg); + $this->assertStringContainsString('SHOULD BE A STRING', $fabric->contracts[0]->error_msg); + } + + /** + * @test + */ + public function testFillMultiContractWithEmptyContentSetsContractError() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->contracts = [ + new HTTPRequestContract('https://example.com/file1.gz') + ]; + + $results = [ + 'https://example.com/file1.gz' => '' + ]; + + $fabric->fillMultiContract($results); + + $this->assertFalse($fabric->contracts[0]->success); + $this->assertNotNull($fabric->contracts[0]->error_msg); + $this->assertStringContainsString('SHOULD BE NOT EMPTY', $fabric->contracts[0]->error_msg); + } + + /** + * @test + */ + public function testFillMultiContractWithPartialSuccessSuggestsBatchReduce() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->contracts = [ + new HTTPRequestContract('https://example.com/file1.gz'), + new HTTPRequestContract('https://example.com/file2.gz'), + new HTTPRequestContract('https://example.com/file3.gz') + ]; + + $results = [ + 'https://example.com/file1.gz' => 'content1', + 'https://example.com/file3.gz' => 'content3' + ]; + + $fabric->fillMultiContract($results); + + $this->assertEquals(2, $fabric->suggest_batch_reduce_to); + $this->assertTrue($fabric->process_done); + } + + /** + * @test + */ + public function testFillMultiContractWithAllFailedSuggestsMinimumBatchSize() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->contracts = [ + new HTTPRequestContract('https://example.com/file1.gz'), + new HTTPRequestContract('https://example.com/file2.gz') + ]; + + $results = []; + + $fabric->fillMultiContract($results); + + $this->assertEquals(2, $fabric->suggest_batch_reduce_to); + $this->assertTrue($fabric->process_done); + } + + /** + * @test + */ + public function testFillMultiContractWithAllSuccessDoesNotSuggestBatchReduce() + { + $fabric = new HTTPMultiRequestFactory(); + + $fabric->contracts = [ + new HTTPRequestContract('https://example.com/file1.gz'), + new HTTPRequestContract('https://example.com/file2.gz') + ]; + + $results = [ + 'https://example.com/file1.gz' => 'content1', + 'https://example.com/file2.gz' => 'content2' + ]; + + $fabric->fillMultiContract($results); + + $this->assertFalse($fabric->suggest_batch_reduce_to); + $this->assertTrue($fabric->process_done); + } + + /** + * @test + */ + public function testGetContractsErrorsReturnsFormattedErrors() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->error_msg = 'Connection timeout'; + + $contract2 = new HTTPRequestContract('https://example.com/file2.gz'); + $contract2->success = true; + $contract2->content = 'content'; + + $contract3 = new HTTPRequestContract('https://example.com/file3.gz'); + $contract3->error_msg = '404 Not Found'; + + $fabric->contracts = [$contract1, $contract2, $contract3]; + + $errors = $fabric->getContractsErrors(); + + $this->assertIsString($errors); + $this->assertStringContainsString('file1.gz', $errors); + $this->assertStringContainsString('Connection timeout', $errors); + $this->assertStringContainsString('file3.gz', $errors); + $this->assertStringContainsString('404 Not Found', $errors); + $this->assertStringNotContainsString('file2', $errors); + } + + /** + * @test + */ + public function testGetContractsErrorsReturnsFalseWhenNoErrors() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content1'; + + $contract2 = new HTTPRequestContract('https://example.com/file2.gz'); + $contract2->success = true; + $contract2->content = 'content2'; + + $fabric->contracts = [$contract1, $contract2]; + + $errors = $fabric->getContractsErrors(); + + $this->assertFalse($errors); + } + + /** + * @test + */ + public function testWriteSuccessURLsContentWritesFiles() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content1'; + + $contract2 = new HTTPRequestContract('https://example.com/file2.gz'); + $contract2->success = true; + $contract2->content = 'content2'; + + $fabric->contracts = [$contract1, $contract2]; + + $result = $fabric->writeSuccessURLsContent($this->testFolder); + + $this->assertIsArray($result); + $this->assertCount(2, $result); + $this->assertFileExists($this->testFolder . 'file1.gz'); + $this->assertFileExists($this->testFolder . 'file2.gz'); + $this->assertEquals('content1', file_get_contents($this->testFolder . 'file1.gz')); + $this->assertEquals('content2', file_get_contents($this->testFolder . 'file2.gz')); + } + + /** + * @test + */ + public function testWriteSuccessURLsContentSkipsFailedContracts() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content1'; + + $contract2 = new HTTPRequestContract('https://example.com/file2.gz'); + $contract2->success = false; + + $contract3 = new HTTPRequestContract('https://example.com/file3.gz'); + $contract3->success = true; + $contract3->content = 'content3'; + + $fabric->contracts = [$contract1, $contract2, $contract3]; + + $result = $fabric->writeSuccessURLsContent($this->testFolder); + + $this->assertIsArray($result); + $this->assertCount(2, $result); + $this->assertFileExists($this->testFolder . 'file1.gz'); + $this->assertFileNotExists($this->testFolder . 'file2.gz'); + $this->assertFileExists($this->testFolder . 'file3.gz'); + } + + /** + * @test + */ + public function testWriteSuccessURLsContentReturnsErrorWhenDirectoryNotExists() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content1'; + + $fabric->contracts = [$contract1]; + + $result = $fabric->writeSuccessURLsContent('/nonexistent/path/'); + + $this->assertIsString($result); + $this->assertStringContainsString('CAN NOT WRITE TO DIRECTORY', $result); + } + + /** + * @test + */ + public function testWriteSuccessURLsContentReturnsErrorWhenDirectoryNotWritable() + { + $fabric = new HTTPMultiRequestFactory(); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content1'; + + $fabric->contracts = [$contract1]; + + // Use root directory which is typically not writable + $result = $fabric->writeSuccessURLsContent('/nonexist'); + + $this->assertIsString($result); + $this->assertStringContainsString('CAN NOT WRITE', $result); + } + + /** + * @test + */ + public function testSetMultiContractResetsStateOnEachCall() + { + $fabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->setMethods(['executeMultiContract']) + ->getMock(); + + // First call + $fabric->setMultiContract(['https://example.com/file1.gz']); + $this->assertCount(1, $fabric->contracts); + $fabric->process_done = true; + $fabric->suggest_batch_reduce_to = 5; + $fabric->error_msg = 'some error'; + + // Second call should reset everything + $fabric->setMultiContract(['https://example.com/file2.gz', 'https://example.com/file3.gz']); + + $this->assertCount(2, $fabric->contracts); + $this->assertFalse($fabric->suggest_batch_reduce_to); + $this->assertNull($fabric->error_msg); + $this->assertFalse($fabric->process_done); + } + + /** + * @test + */ + public function testSetMultiContractReturnsItself() + { + $fabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->setMethods(['executeMultiContract']) + ->getMock(); + + $result = $fabric->setMultiContract(['https://example.com/file1.gz']); + + $this->assertInstanceOf(HTTPMultiRequestFactory::class, $result); + $this->assertSame($fabric, $result); + } + + /** + * @test + */ + public function testFillMultiContractReturnsItself() + { + $fabric = new HTTPMultiRequestFactory(); + + $result = $fabric->fillMultiContract([]); + + $this->assertInstanceOf(HTTPMultiRequestFactory::class, $result); + $this->assertSame($fabric, $result); + } + + /** + * @test + */ + public function testWriteSuccessURLsContentHandlesFileWriteFailure() + { + $fabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->setMethods(['writeFile']) + ->getMock(); + + // Mock writeFile to return false (simulating write failure) + $fabric->expects($this->once()) + ->method('writeFile') + ->willReturn(false); + + $contract1 = new HTTPRequestContract('https://example.com/file1.gz'); + $contract1->success = true; + $contract1->content = 'content1'; + + $fabric->contracts = [$contract1]; + + $result = $fabric->writeSuccessURLsContent($this->testFolder); + + $this->assertIsString($result); + $this->assertStringContainsString('CAN NOT WRITE TO FILE', $result); + } +} diff --git a/tests/ApbctWP/HTTP/TestHTTPRequestContract.php b/tests/ApbctWP/HTTP/TestHTTPRequestContract.php new file mode 100644 index 000000000..47a1a4dd2 --- /dev/null +++ b/tests/ApbctWP/HTTP/TestHTTPRequestContract.php @@ -0,0 +1,70 @@ +assertEquals($url, $contract->url); + $this->assertEquals('', $contract->content); + $this->assertFalse($contract->success); + $this->assertNull($contract->error_msg); + } + + /** + * @test + */ + public function testPropertiesCanBeModified() + { + $contract = new HTTPRequestContract('https://example.com/file1.gz'); + + $contract->content = 'test content'; + $contract->success = true; + $contract->error_msg = 'test error'; + + $this->assertEquals('test content', $contract->content); + $this->assertTrue($contract->success); + $this->assertEquals('test error', $contract->error_msg); + } + + /** + * @test + */ + public function testSuccessStateWithContent() + { + $contract = new HTTPRequestContract('https://example.com/file1.gz'); + + $contract->success = true; + $contract->content = 'downloaded content'; + + $this->assertTrue($contract->success); + $this->assertNotEmpty($contract->content); + $this->assertNull($contract->error_msg); + } + + /** + * @test + */ + public function testFailureStateWithError() + { + $contract = new HTTPRequestContract('https://example.com/file1.gz'); + + $contract->success = false; + $contract->error_msg = 'Connection timeout'; + + $this->assertFalse($contract->success); + $this->assertEquals('Connection timeout', $contract->error_msg); + $this->assertEmpty($contract->content); + } +} diff --git a/tests/ApbctWP/HTTP/TestSFWFilesDownloader.php b/tests/ApbctWP/HTTP/TestSFWFilesDownloader.php new file mode 100644 index 000000000..b8e54e4f6 --- /dev/null +++ b/tests/ApbctWP/HTTP/TestSFWFilesDownloader.php @@ -0,0 +1,440 @@ +apbctBackup = $apbct; + + $this->testFolder = sys_get_temp_dir() . '/test_sfw_' . time() . '/'; + if (!is_dir($this->testFolder)) { + mkdir($this->testFolder, 0777, true); + } + + $apbct = new State('cleantalk', array('settings', 'data', 'errors', 'remote_calls', 'stats', 'fw_stats')); + $apbct->data = ['sfw_update__batch_size' => 10]; + $apbct->fw_stats = ['updating_folder' => $this->testFolder]; + $apbct->save = function($key) {}; + } + + protected function tearDown() + { + parent::tearDown(); + global $apbct; + + if (is_dir($this->testFolder)) { + $files = glob($this->testFolder . '/*'); + if ($files) { + foreach ($files as $file) { + if (is_file($file)) { + unlink($file); + } + } + } + rmdir($this->testFolder); + } + + $apbct = $this->apbctBackup; + } + + /** + * @test + */ + public function testReturnsErrorWhenFolderNotWritable() + { + global $apbct; + $apbct->fw_stats['updating_folder'] = '/nonexistent/path/'; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->getMock(); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles(['https://example.com/file1.gz'], false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('NOT WRITABLE', $result['error']); + $this->assertArrayNotHasKey('update_args', $result); + } + + /** + * @test + */ + public function testReturnsErrorWhenUrlsNotArray() + { + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->getMock(); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles('NOT AN ARRAY', false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('SHOULD BE AN ARRAY', $result['error']); + } + + /** + * @test + */ + public function testReturnsSuccessStageWhenEmptyUrlsArray() + { + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->getMock(); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles([], false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('next_stage', $result); + $this->assertEquals('apbct_sfw_update__create_tables', $result['next_stage']['name']); + } + + /** + * @test + */ + public function testReturnsTrueWhenEmptyUrlsAndDirectUpdateTrue() + { + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->getMock(); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles([], true, 0); + + $this->assertTrue((bool)$result); + } + + /** + * @test + */ + public function testReturnsErrorWhenContractProcessNotDone() + { + $urls = ['https://example.com/file1.gz']; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract']) + ->getMock(); + + $mockFabric->expects($this->once()) + ->method('setMultiContract') + ->with($urls) + ->willReturnCallback(function() use ($mockFabric) { + $mockFabric->process_done = false; + $mockFabric->error_msg = 'CONTRACT PROCESSING FAILED'; + return $mockFabric; + }); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles($urls, false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('CONTRACT PROCESSING FAILED', $result['error']); + } + + /** + * @test + */ + public function testReturnsRepeatStageWhenSomeFilesFailedToDownload() + { + global $apbct; + $apbct->fw_stats['multi_request_batch_size'] = 10; + + $urls = [ + 'https://example.com/file1.gz', + 'https://example.com/file2.gz', + 'https://example.com/file3.gz' + ]; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract', 'getFailedURLs', 'writeSuccessURLsContent']) + ->getMock(); + + $mockFabric->method('setMultiContract') + ->willReturnCallback(function() use ($mockFabric) { + $mockFabric->process_done = true; + $mockFabric->suggest_batch_reduce_to = 2; + return $mockFabric; + }); + + $mockFabric->method('getFailedURLs') + ->willReturn(['https://example.com/file2.gz']); + + $mockFabric->method('writeSuccessURLsContent') + ->willReturn(['https://example.com/file1.gz', 'https://example.com/file3.gz']); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles($urls, false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('NOT COMPLETED, TRYING AGAIN', $result['error']); + $this->assertArrayHasKey('update_args', $result); + $this->assertEquals(['https://example.com/file2.gz'], $result['update_args']['args']); + $this->assertEquals(2, $apbct->fw_stats['multi_request_batch_size']); + } + + /** + * @test + */ + public function testReturnsErrorWhenWriteToFileSystemFails() + { + $urls = ['https://example.com/file1.gz']; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract', 'getFailedURLs', 'writeSuccessURLsContent']) + ->getMock(); + + $mockFabric->method('setMultiContract') + ->willReturnCallback(function() use ($mockFabric) { + $mockFabric->process_done = true; + return $mockFabric; + }); + + $mockFabric->method('getFailedURLs') + ->willReturn([]); + + $mockFabric->method('writeSuccessURLsContent') + ->willReturn('CAN NOT WRITE TO FILE: /test/file1.gz'); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles($urls, false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('CAN NOT WRITE TO FILE', $result['error']); + $this->assertStringContainsString('/test/file1.gz', $result['error']); + } + + /** + * @test + */ + public function testReturnsNextStageWhenAllFilesDownloadedSuccessfully() + { + $urls = [ + 'https://example.com/file1.gz', + 'https://example.com/file2.gz' + ]; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract', 'getFailedURLs', 'writeSuccessURLsContent']) + ->getMock(); + + $mockFabric->method('setMultiContract') + ->willReturnCallback(function() use ($mockFabric) { + $mockFabric->process_done = true; + $mockFabric->suggest_batch_reduce_to = false; + return $mockFabric; + }); + + $mockFabric->method('getFailedURLs') + ->willReturn([]); + + $mockFabric->method('writeSuccessURLsContent') + ->willReturn($urls); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles($urls, false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('next_stage', $result); + $this->assertEquals('apbct_sfw_update__create_tables', $result['next_stage']['name']); + } + + /** + * @test + */ + public function testProcessesUrlsInBatchesAccordingToBatchSize() + { + global $apbct; + $apbct->fw_stats['multi_request_batch_size'] = 3; + + $urls = [ + 'https://example.com/file1.gz', + 'https://example.com/file2.gz', + 'https://example.com/file3.gz', + 'https://example.com/file4.gz', + 'https://example.com/file5.gz' + ]; + + $callCount = 0; + $receivedBatches = []; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract', 'getFailedURLs', 'writeSuccessURLsContent']) + ->getMock(); + + $mockFabric->method('setMultiContract') + ->willReturnCallback(function($batchUrls) use ($mockFabric, &$callCount, &$receivedBatches) { + $callCount++; + $receivedBatches[] = $batchUrls; + $mockFabric->process_done = true; + $mockFabric->suggest_batch_reduce_to = false; + return $mockFabric; + }); + + $mockFabric->method('getFailedURLs') + ->willReturn([]); + + $mockFabric->method('writeSuccessURLsContent') + ->willReturnCallback(function() use (&$receivedBatches, &$callCount) { + return $receivedBatches[$callCount - 1]; + }); + + $downloader = new SFWFilesDownloader($mockFabric); + $downloader->downloadFiles($urls, false, 0); + + $this->assertEquals(2, $callCount); + $this->assertCount(3, $receivedBatches[0]); + $this->assertCount(2, $receivedBatches[1]); + } + + /** + * @test + */ + public function testReducesBatchSizeToMinimumWhenMultipleSuggestions() + { + global $apbct; + $apbct->fw_stats['multi_request_batch_size'] = 10; + + $urls = []; + for ($i = 0; $i < 20; $i++) { + $urls[] = 'https://example.com/file' . $i . '.gz'; + } + + $callCount = 0; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract', 'getFailedURLs', 'writeSuccessURLsContent']) + ->getMock(); + + $mockFabric->method('setMultiContract') + ->willReturnCallback(function($batchUrls) use ($mockFabric, &$callCount) { + $callCount++; + $mockFabric->process_done = true; + $mockFabric->suggest_batch_reduce_to = $callCount === 1 ? 7 : 5; + return $mockFabric; + }); + + $mockFabric->method('getFailedURLs') + ->willReturnCallback(function() use (&$callCount, $urls) { + $batchStart = ($callCount - 1) * 10; + return [$urls[$batchStart]]; + }); + + $mockFabric->method('writeSuccessURLsContent') + ->willReturnCallback(function() use (&$callCount, $urls) { + $batchStart = ($callCount - 1) * 10; + $batchSize = min(10, count($urls) - $batchStart); + $result = []; + for ($i = 1; $i < $batchSize; $i++) { + $result[] = $urls[$batchStart + $i]; + } + return $result; + }); + + $downloader = new SFWFilesDownloader($mockFabric); + $downloader->downloadFiles($urls, false, 0); + + $this->assertEquals(5, $apbct->fw_stats['multi_request_batch_size']); + } + + /** + * @test + */ + public function testReturnsErrorWhenNotAllFilesDownloadedAfterBatches() + { + $urls = [ + 'https://example.com/file1.gz', + 'https://example.com/file2.gz' + ]; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract', 'getFailedURLs', 'writeSuccessURLsContent', 'getContractsErrors']) + ->getMock(); + + $mockFabric->method('setMultiContract') + ->willReturnCallback(function() use ($mockFabric) { + $mockFabric->process_done = true; + return $mockFabric; + }); + + $mockFabric->method('getFailedURLs') + ->willReturn([]); + + $mockFabric->method('writeSuccessURLsContent') + ->willReturn(['https://example.com/file1.gz']); + + $mockFabric->method('getContractsErrors') + ->willReturn('[url1]:[error1],[url2]:[error2]'); + + $downloader = new SFWFilesDownloader($mockFabric); + $result = $downloader->downloadFiles($urls, false, 0); + + $this->assertIsArray($result); + $this->assertArrayHasKey('error', $result); + $this->assertStringContainsString('NOT COMPLETED - STOP UPDATE', $result['error']); + $this->assertStringContainsString('[url1]:[error1],[url2]:[error2]', $result['error']); + } + + /** + * @test + */ + public function testRemovesDuplicateUrlsAndResetsKeys() + { + $urls = [ + 5 => 'https://example.com/file1.gz', + 7 => 'https://example.com/file1.gz', + 9 => 'https://example.com/file2.gz' + ]; + + $receivedUrls = null; + + $mockFabric = $this->getMockBuilder(HTTPMultiRequestFactory::class) + ->disableOriginalConstructor() + ->setMethods(['setMultiContract', 'getFailedURLs', 'writeSuccessURLsContent']) + ->getMock(); + + $mockFabric->method('setMultiContract') + ->willReturnCallback(function($batchUrls) use ($mockFabric, &$receivedUrls) { + $receivedUrls = $batchUrls; + $mockFabric->process_done = true; + return $mockFabric; + }); + + $mockFabric->method('getFailedURLs') + ->willReturn([]); + + $mockFabric->method('writeSuccessURLsContent') + ->willReturn(['https://example.com/file1.gz', 'https://example.com/file2.gz']); + + $downloader = new SFWFilesDownloader($mockFabric); + $downloader->downloadFiles($urls, false, 0); + + $this->assertCount(2, $receivedUrls); + $this->assertEquals(['https://example.com/file1.gz', 'https://example.com/file2.gz'], $receivedUrls); + } +}