Skip to content

upd-sfw-upd.ag #715

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alexandergull wants to merge 5 commits into
base: dev
Choose a base branch
from
Open

upd-sfw-upd.ag #715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
069aec3
Upd. Code. SFW Update. HTTP multi request refactored.
svfcode Dec 27, 2025
07d1b68
Merge branch 'dev' into upd-sfw-upd.ag
alexandergull Jan 19, 2026
2b3416e
Test codecov. #1
alexandergull Jan 19, 2026
258380c
Test codecov - master. #2
alexandergull Jan 19, 2026
6b63bc9
Fix. Codecov.
alexandergull Jan 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 5 additions & 84 deletions cleantalk.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1531,92 +1531,13 @@ function apbct_sfw_update__get_multifiles_of_type(array $params)

/**
* Queue stage. Do load multifiles with networks on their urls.
* @param $urls
* @return array|array[]|bool|string|string[]
* @param $all_urls
* @return array|true
*/
function apbct_sfw_update__download_files($urls, $direct_update = false)
function apbct_sfw_update__download_files($all_urls, $direct_update = false)
{
global $apbct;

sleep(3);

if ( ! is_writable($apbct->fw_stats['updating_folder']) ) {
return array('error' => 'SFW update folder is not writable.');
}

//Reset keys
$urls = array_values(array_unique($urls));

$results = array();
$batch_size = 10;

/**
* Reduce batch size of curl multi instanced
*/
if (defined('APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE')) {
if (
is_int(APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE) &&
APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE > 0 &&
APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE < 10
) {
$batch_size = APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE;
};
}

$total_urls = count($urls);
$batches = ceil($total_urls / $batch_size);

for ($i = 0; $i < $batches; $i++) {
$batch_urls = array_slice($urls, $i * $batch_size, $batch_size);
if (!empty($batch_urls)) {
$http_results = Helper::httpMultiRequest($batch_urls, $apbct->fw_stats['updating_folder']);
if (is_array($http_results)) {
$results = array_merge($results, $http_results);
}
// to handle case if we request only one url, then Helper::httpMultiRequest returns string 'success' instead of array
if (count($batch_urls) === 1 && $http_results === 'success') {
$results = array_merge($results, $batch_urls);
}
}
}

$results = TT::toArray($results);
$count_urls = count($urls);
$count_results = count($results);

if ( empty($results['error']) && ($count_urls === $count_results) ) {
if ( $direct_update ) {
return true;
}
$download_again = array();
$results = array_values($results);
for ( $i = 0; $i < $count_results; $i++ ) {
if ( $results[$i] === 'error' ) {
$download_again[] = $urls[$i];
}
}

if ( count($download_again) !== 0 ) {
return array(
'error' => 'Files download not completed.',
'update_args' => array(
'args' => $download_again
)
);
}

return array(
'next_stage' => array(
'name' => 'apbct_sfw_update__create_tables'
)
);
}

if ( ! empty($results['error']) ) {
return $results;
}

return array('error' => 'Files download not completed.');
$downloader = new \Cleantalk\ApbctWP\Firewall\SFWFilesDownloader();
return $downloader->downloadFiles($all_urls, $direct_update);
}

/**
Expand Down
205 changes: 205 additions & 0 deletions lib/Cleantalk/ApbctWP/Firewall/SFWFilesDownloader.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
<?php

namespace Cleantalk\ApbctWP\Firewall;

use Cleantalk\ApbctWP\HTTP\HTTPMultiRequestFactory;
use Cleantalk\Common\TT;

/**
* Class SFWFilesDownloader
*
* Handles downloading of SpamFireWall data files from remote servers.
* Manages batch processing, retry logic, and error handling for file downloads.
*/
class SFWFilesDownloader
{
/**
* HTTP multi-request fabric instance
*
* @var HTTPMultiRequestFactory
*/
private $http_multi_request_factory;

/**
* @var string
*/
private $deafult_error_prefix;

/**
* @var string
*/
private $deafult_error_content = 'UNKNOWN ERROR';

/**
* SFWFilesDownloader constructor
*
* @param HTTPMultiRequestFactory|null $factory Optional. Custom fabric instance for dependency injection.
*/
public function __construct($factory = null)
{
$this->deafult_error_prefix = basename(__CLASS__) . ': ';
$this->http_multi_request_factory = $factory ?: new HTTPMultiRequestFactory();
}

/**
* Downloads SFW data files from provided URLs with batch processing and retry logic
*
* Downloads files in batches to avoid server overload. Automatically retries failed downloads
* with reduced batch size if necessary. Validates write permissions and URL format before processing.
*
* @param array|mixed $all_urls List of URLs to download files from
* @param bool $direct_update Optional. If true, returns boolean result. If false, returns stage info array.
* @param int sleep Pause in seconds before multi contracts run, default is 3
*
* @return true|array True on success (direct update mode), or array with 'next_stage' key,
* or array with 'error' key on failure, or array with 'update_args' for retry.
*/
public function downloadFiles($all_urls, $direct_update = false, $sleep = 3)
{
global $apbct;

// Delay to prevent server overload
sleep($sleep);

// Validate write permissions for update folder
if ( ! is_writable($apbct->fw_stats['updating_folder']) ) {
return $this->responseStopUpdate('SFW UPDATE FOLDER IS NOT WRITABLE.');
}

// Validate URLs parameter type
if ( ! is_array($all_urls) ) {
return $this->responseStopUpdate('URLS LIST SHOULD BE AN ARRAY');
}

// Remove duplicates and reset array keys to sequential integers
$all_urls = array_values(array_unique($all_urls));

// Get current batch size from settings or default
$work_batch_size = SFWUpdateHelper::getSFWFilesBatchSize();

// Initialize batch processing variables
$total_urls = count($all_urls);
$batches = ceil($total_urls / $work_batch_size);
$download_again = [];
$written_urls = [];

// Get or set default batch size for retry attempts
$on_repeat_batch_size = !empty($apbct->data['sfw_update__batch_size'])
? TT::toInt($apbct->data['sfw_update__batch_size'])
: 10;

// Process URLs in batches
for ($i = 0; $i < $batches; $i++) {
// Extract current batch of URLs
$current_batch_urls = array_slice($all_urls, $i * $work_batch_size, $work_batch_size);

if (!empty($current_batch_urls)) {
// Execute multi-request for current batch
$multi_request_contract = $this->http_multi_request_factory->setMultiContract($current_batch_urls);

// Critical error: contract processing failed, stop update immediately
if (!$multi_request_contract->process_done) {
$error = !empty($multi_request_contract->error_msg) ? $multi_request_contract->error_msg : 'UNKNOWN ERROR';
return $this->responseStopUpdate($error);
}

// Handle failed downloads in this batch
if (!empty($multi_request_contract->getFailedURLs())) {
// Reduce batch size for retry if fabric suggests it
if ($multi_request_contract->suggest_batch_reduce_to) {
$on_repeat_batch_size = min($on_repeat_batch_size, $multi_request_contract->suggest_batch_reduce_to);
}
// Collect failed URLs for retry
$download_again = array_merge($download_again, $multi_request_contract->getFailedURLs());
}

// Write successfully downloaded content to files
$write_result = $multi_request_contract->writeSuccessURLsContent($apbct->fw_stats['updating_folder']);

// File write error occurred, stop update
if (is_string($write_result)) {
return $this->responseStopUpdate($write_result);
}

// Track successfully written URLs
$written_urls = array_merge($written_urls, $write_result);
}
}

// Some downloads failed, schedule retry with adjusted batch size
if (!empty($download_again)) {
$apbct->fw_stats['multi_request_batch_size'] = $on_repeat_batch_size;
$apbct->save('data');
return $this->responseRepeatStage('FILES DOWNLOAD NOT COMPLETED, TRYING AGAIN', $download_again);
}

// Verify all URLs were successfully downloaded and written
if (empty(array_diff($all_urls, $written_urls))) {
return $this->responseSuccess($direct_update);
}

// Download incomplete with no retry - collect error information
$last_contract_errors = isset($multi_request_contract) && $multi_request_contract->getContractsErrors()
? $multi_request_contract->getContractsErrors()
: 'no known contract errors';

$error = 'FILES DOWNLOAD NOT COMPLETED - STOP UPDATE, ERRORS: ' . $last_contract_errors;
return $this->responseStopUpdate($error);
}

/**
* Creates error response to stop the update process
*
* @param string $message Error message describing why update was stopped
*
* @return array Error response array with 'error' key
*/
private function responseStopUpdate($message): array
{
$message = is_string($message) ? $message : $this->deafult_error_content;
$message = $this->deafult_error_prefix . $message;
return [
'error' => $message
];
}

/**
* Creates response to repeat current stage with modified arguments
*
* Used when downloads partially failed and should be retried with
* potentially reduced batch size or different parameters.
*
* @param string $message Descriptive message about why stage needs repeating
* @param array $args Arguments for retry attempt (typically failed URLs)
*
* @return array Response array with 'error' message and 'update_args' for retry
*/
private function responseRepeatStage($message, $args): array
{
$args = is_array($args) ? $args : [];
$message = is_string($message) ? $message : $this->deafult_error_content;
$message = $this->deafult_error_prefix . $message;
return [
'error' => $message,
'update_args' => [
'args' => $args
],
];
}

/**
* Creates success response to proceed to next stage or complete update
*
* @param bool $direct_update If true, returns simple boolean. If false, returns stage transition array.
*
* @return true|array True for direct update mode, or array with 'next_stage' key for staged updates
*/
private function responseSuccess($direct_update)
{
return $direct_update ? true : [
'next_stage' => array(
'name' => 'apbct_sfw_update__create_tables'
)
];
}
}
25 changes: 25 additions & 0 deletions lib/Cleantalk/ApbctWP/Firewall/SFWUpdateHelper.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -578,4 +578,29 @@ public static function fallback()
*/
apbct_sfw_update__create_tables();
}

public static function getSFWFilesBatchSize()
{
global $apbct;
$work_batch_size = 10;

/**
* Reduce batch size of curl multi instanced
*/
if (defined('APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE')) {
if (
is_int(APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE) &&
APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE > 1 &&
APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE < 10
) {
$work_batch_size = APBCT_SERVICE__SFW_UPDATE_CURL_MULTI_BATCH_SIZE;
};
}

$work_batch_size = !empty($apbct->fw_stats['multi_request_batch_size'])
? TT::toInt($apbct->fw_stats['multi_request_batch_size'], $work_batch_size)
: $work_batch_size;

return $work_batch_size;
}
}
Loading