File: /home/beaupptk/birthkumar.shop/wp-content/plugins/litespeed-cache/src/crawler.cls.php
<?php
/**
* The crawler class.
*
* @package LiteSpeed
* @since 1.1.0
*/
namespace LiteSpeed;
defined( 'WPINC' ) || exit();
/**
* Handles sitemap crawling, blacklisting, and async operations.
*/
class Crawler extends Root {
const LOG_TAG = '🕸️';
const TYPE_REFRESH_MAP = 'refresh_map';
const TYPE_EMPTY = 'empty';
const TYPE_BLACKLIST_EMPTY = 'blacklist_empty';
const TYPE_BLACKLIST_DEL = 'blacklist_del';
const TYPE_BLACKLIST_ADD = 'blacklist_add';
const TYPE_START = 'start';
const TYPE_RESET = 'reset';
const USER_AGENT = 'lscache_walker';
const FAST_USER_AGENT = 'lscache_runner';
const CHUNKS = 10000;
const STATUS_WAIT = 'W';
const STATUS_HIT = 'H';
const STATUS_MISS = 'M';
const STATUS_BLACKLIST = 'B';
const STATUS_NOCACHE = 'N';
/**
* Sitemeta file slug.
*
* @var string
*/
private $_sitemeta = 'meta.data';
/**
* Reset file full path.
*
* @var string
*/
private $_resetfile;
/**
* Reason that ended current run.
*
* @var string
*/
private $_end_reason;
/**
* Number of CPU cores.
*
* @var int
*/
private $_ncpu = 1;
/**
* Server IP set in settings.
*
* @var string|null
*/
private $_server_ip;
/**
* Crawler configuration.
*
* @var array
*/
private $_crawler_conf = [
'cookies' => [],
'headers' => [],
'ua' => '',
];
/**
* Built crawler variants.
*
* @var array<int,array>
*/
private $_crawlers = [];
/**
* Current allowed worker threads.
*
* @var int
*/
private $_cur_threads = -1;
/**
* Max timestamp to run until.
*
* @var int
*/
private $_max_run_time;
/**
* Last time threads were adjusted.
*
* @var int
*/
private $_cur_thread_time;
/**
* Map-status list to batch-save.
*
* @var array
*/
private $_map_status_list = [
'H' => [],
'M' => [],
'B' => [],
'N' => [],
];
/**
* Summary cache.
*
* @var array
*/
protected $_summary;
/**
* Initialize crawler, assign sitemap path.
*
* @since 1.1.0
*/
public function __construct() {
if ( is_multisite() ) {
$this->_sitemeta = 'meta' . get_current_blog_id() . '.data';
}
$this->_resetfile = LITESPEED_STATIC_DIR . '/crawler/' . $this->_sitemeta . '.reset';
$this->_summary = self::get_summary();
$this->_ncpu = $this->_get_server_cpu();
$this->_server_ip = $this->conf( Base::O_SERVER_IP );
self::debug( 'Init w/ CPU cores=' . $this->_ncpu );
}
/**
* Try get server CPUs.
*
* @since 5.2
* @return int Number of cores detected.
*/
private function _get_server_cpu() {
$cpuinfo_file = '/proc/cpuinfo';
$setting_open_dir = ini_get( 'open_basedir' );
if ( $setting_open_dir ) {
return 1; // Server has limit.
}
try {
// phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged
if (!@is_file($cpuinfo_file)) {
return 1;
}
} catch ( \Exception $e ) {
return 1;
}
// Local system read; no WP alternative. Suppress sniff.
// phpcs:ignore WordPress.WP.AlternativeFunctions.file_get_contents_file_get_contents
$cpuinfo = file_get_contents( $cpuinfo_file );
preg_match_all( '/^processor/m', $cpuinfo, $matches );
$cnt = isset( $matches[0] ) ? count( $matches[0] ) : 0;
return $cnt ? $cnt : 1;
}
/**
* Check whether the current crawler is active.
*
* @since 4.3
* @param int $curr Crawler index.
* @return bool Active state.
*/
public function is_active( $curr ) {
$bypass_list = self::get_option( 'bypass_list', [] );
return ! in_array( (int) $curr, $bypass_list, true );
}
/**
* Toggle the current crawler's active state and return the updated state.
*
* @since 4.3
* @param int $curr Crawler index.
* @return bool True if turned on, false if turned off.
*/
public function toggle_activeness( $curr ) {
$bypass_list = self::get_option( 'bypass_list', [] );
if ( in_array( (int) $curr, $bypass_list, true ) ) {
// Remove it.
$key = array_search( (int) $curr, $bypass_list, true );
if ( false !== $key ) {
unset( $bypass_list[ $key ] );
$bypass_list = array_values( $bypass_list );
self::update_option( 'bypass_list', $bypass_list );
}
return true;
}
// Add it.
$bypass_list[] = (int) $curr;
self::update_option( 'bypass_list', $bypass_list );
return false;
}
/**
* Clear bypassed list.
*
* @since 4.3
* @access public
* @return void
*/
public function clear_disabled_list() {
self::update_option( 'bypass_list', [] );
$msg = __( 'Crawler disabled list is cleared! All crawlers are set to active! ', 'litespeed-cache' );
Admin_Display::note( $msg );
self::debug( 'All crawlers are set to active...... ' );
}
/**
* Overwrite get_summary to init elements.
*
* @since 3.0
* @access public
*
* @param string|false $field Field name to fetch or false to get all.
* @return mixed Summary value/array or null if not found.
*/
public static function get_summary( $field = false ) {
$_default = [
'list_size' => 0,
'last_update_time' => 0,
'curr_crawler' => 0,
'curr_crawler_beginning_time' => 0,
'last_pos' => 0,
'last_count' => 0,
'last_crawled' => 0,
'last_start_time' => 0,
'last_status' => '',
'is_running' => 0,
'end_reason' => '',
'meta_save_time' => 0,
'pos_reset_check' => 0,
'done' => 0,
'this_full_beginning_time' => 0,
'last_full_time_cost' => 0,
'last_crawler_total_cost' => 0,
'crawler_stats' => [], // this will store all crawlers hit/miss crawl status.
];
wp_cache_delete( 'alloptions', 'options' ); // ensure the summary is current.
$summary = parent::get_summary();
$summary = array_merge( $_default, $summary );
if ( false === $field ) {
return $summary;
}
if ( array_key_exists( $field, $summary ) ) {
return $summary[ $field ];
}
return null;
}
/**
* Overwrite save_summary.
*
* @since 3.0
* @access public
*
* @param array|false $data Data to save or false to save current.
* @param bool $reload Whether to reload after saving.
* @param bool $overwrite Whether to overwrite completely.
* @return void
*/
public static function save_summary( $data = false, $reload = false, $overwrite = false ) {
$instance = self::cls();
$instance->_summary['meta_save_time'] = time();
if ( false === $data ) {
$data = $instance->_summary;
}
parent::save_summary( $data, $reload, $overwrite );
File::save( LITESPEED_STATIC_DIR . '/crawler/' . $instance->_sitemeta, wp_json_encode( $data ), true );
}
/**
* Cron start async crawling.
*
* @since 5.5
* @return void
*/
public static function start_async_cron() {
Task::async_call( 'crawler' );
}
/**
* Manually start async crawling.
*
* @since 5.5
* @return void
*/
public static function start_async() {
Task::async_call( 'crawler_force' );
$msg = __( 'Started async crawling', 'litespeed-cache' );
Admin_Display::success( $msg );
}
/**
* Ajax crawl handler.
*
* @since 5.5
* @param bool $manually_run Whether manually triggered.
* @return void
*/
public static function async_handler( $manually_run = false ) {
self::debug( '------------async-------------start_async_handler' );
self::start( (bool) $manually_run );
}
/**
* Proceed crawling.
*
* @since 1.1.0
* @access public
*
* @param bool $manually_run Whether manually triggered.
* @return bool|void
*/
public static function start( $manually_run = false ) {
if ( ! Router::can_crawl() ) {
self::debug( '......crawler is NOT allowed by the server admin......' );
return false;
}
if ( $manually_run ) {
self::debug( '......crawler manually ran......' );
}
self::cls()->_crawl_data( (bool) $manually_run );
}
/**
* Crawling start.
*
* @since 1.1.0
* @access private
*
* @param bool $manually_run Whether manually triggered.
* @return void
*/
private function _crawl_data( $manually_run ) {
if ( ! defined( 'LITESPEED_LANE_HASH' ) ) {
define( 'LITESPEED_LANE_HASH', Str::rrand( 8 ) );
}
if ( $this->_check_valid_lane() ) {
$this->_take_over_lane();
} else {
self::debug( '⚠️ lane in use' );
return;
}
self::debug( '......crawler started......' );
// for the first time running.
if ( ! $this->_summary || ! Data::cls()->tb_exist( 'crawler' ) || ! Data::cls()->tb_exist( 'crawler_blacklist' ) ) {
$this->cls( 'Crawler_Map' )->gen();
}
// if finished last time, regenerate sitemap.
if ( 'touchedEnd' === $this->_summary['done'] ) {
// check whole crawling interval.
$last_finished_at = (int) $this->_summary['last_full_time_cost'] + (int) $this->_summary['this_full_beginning_time'];
if ( ! $manually_run && ( time() - $last_finished_at ) < $this->conf( Base::O_CRAWLER_CRAWL_INTERVAL ) ) {
self::debug( 'Cron abort: cache warmed already.' );
$this->Release_lane();
return;
}
self::debug( 'TouchedEnd. regenerate sitemap....' );
$this->cls( 'Crawler_Map' )->gen();
}
$crawlers = $this->list_crawlers();
$crawlers_count = count( $crawlers );
// Skip the crawlers that in bypassed list.
while ( ! $this->is_active( $this->_summary['curr_crawler'] ) && $this->_summary['curr_crawler'] < $crawlers_count ) {
self::debug( 'Skipped the Crawler #' . $this->_summary['curr_crawler'] . ' ......' );
$this->_summary['curr_crawler'] = (int) $this->_summary['curr_crawler'] + 1;
}
if ( $this->_summary['curr_crawler'] >= $crawlers_count ) {
$this->_end_reason = 'end';
$this->_terminate_running();
$this->Release_lane();
return;
}
// In case crawlers are all done but not reload, reload it.
if ( empty( $this->_summary['curr_crawler'] ) || empty( $this->_crawlers[ $this->_summary['curr_crawler'] ] ) ) {
$this->_summary['curr_crawler'] = 0;
$this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ] = [];
}
$res = $this->load_conf();
if ( ! $res ) {
self::debug( 'Load conf failed' );
$this->_terminate_running();
$this->Release_lane();
return;
}
try {
$this->_engine_start();
$this->Release_lane();
} catch ( \Exception $e ) {
self::debug( '🛑 ' . $e->getMessage() );
}
}
/**
* Load conf before running crawler.
*
* @since 3.0
* @access private
* @return bool True on success.
*/
private function load_conf() {
$this->_crawler_conf['base'] = site_url();
$current_crawler = $this->_crawlers[ $this->_summary['curr_crawler'] ];
// Cookies.
foreach ( $current_crawler as $k => $v ) {
if ( 0 !== strpos( $k, 'cookie:' ) ) {
continue;
}
if ( '_null' === $v ) {
continue;
}
$this->_crawler_conf['cookies'][ substr( $k, 7 ) ] = $v;
}
// WebP/AVIF simulation.
if ( ! empty( $current_crawler['webp'] ) ) {
$this->_crawler_conf['headers'][] = 'Accept: image/' . ( 2 === (int) $this->conf( Base::O_IMG_OPTM_WEBP ) ? 'avif' : 'webp' ) . ',*/*';
}
// Mobile crawler.
if ( ! empty( $current_crawler['mobile'] ) ) {
$this->_crawler_conf['ua'] = 'Mobile iPhone';
}
// Limit delay to use server setting.
$this->_crawler_conf['run_delay'] = 500; // microseconds.
if ( defined( 'LITESPEED_CRAWLER_USLEEP' ) && constant( 'LITESPEED_CRAWLER_USLEEP' ) > $this->_crawler_conf['run_delay'] ) {
$this->_crawler_conf['run_delay'] = (int) constant( 'LITESPEED_CRAWLER_USLEEP' );
}
if ( isset( $_SERVER[ Base::ENV_CRAWLER_USLEEP ] ) ) {
$env_usleep = absint( wp_unslash( $_SERVER[ Base::ENV_CRAWLER_USLEEP ] ) );
if ( $env_usleep > (int) $this->_crawler_conf['run_delay'] ) {
$this->_crawler_conf['run_delay'] = $env_usleep;
}
}
$this->_crawler_conf['run_duration'] = $this->get_crawler_duration();
$this->_crawler_conf['load_limit'] = (int) $this->conf( Base::O_CRAWLER_LOAD_LIMIT );
if ( isset( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT_ENFORCE ] ) ) {
$this->_crawler_conf['load_limit'] = absint( wp_unslash( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT_ENFORCE ] ) );
} elseif ( isset( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT ] ) ) {
$env_limit = absint( wp_unslash( $_SERVER[ Base::ENV_CRAWLER_LOAD_LIMIT ] ) );
if ( $env_limit < (int) $this->_crawler_conf['load_limit'] ) {
$this->_crawler_conf['load_limit'] = $env_limit;
}
}
if ( 0 === (int) $this->_crawler_conf['load_limit'] ) {
self::debug( '🛑 Terminated crawler due to load limit set to 0' );
return false;
}
// Role simulation.
if ( ! empty( $current_crawler['uid'] ) ) {
if ( empty( $this->_server_ip ) ) {
self::debug( '🛑 Terminated crawler due to Server IP not set' );
return false;
}
$vary_name = $this->cls( 'Vary' )->get_vary_name();
$vary_val = $this->cls( 'Vary' )->finalize_default_vary( $current_crawler['uid'] );
$this->_crawler_conf['cookies'][ $vary_name ] = $vary_val;
$this->_crawler_conf['cookies']['litespeed_hash'] = Router::cls()->get_hash( $current_crawler['uid'] );
}
return true;
}
/**
* Get crawler duration allowance.
*
* @since 7.0
* @return int Seconds.
*/
public function get_crawler_duration() {
$run_duration = defined( 'LITESPEED_CRAWLER_DURATION' ) ? (int) constant( 'LITESPEED_CRAWLER_DURATION' ) : 900;
if ( $run_duration > 900 ) {
$run_duration = 900; // reset to default value if defined higher than 900 seconds.
}
return $run_duration;
}
/**
* Start crawler.
*
* @since 1.1.0
* @access private
* @return void
*/
private function _engine_start() {
// check current load.
$this->_adjust_current_threads();
if ( 0 === (int) $this->_cur_threads ) {
$this->_end_reason = 'stopped_highload';
self::debug( 'Stopped due to heavy load.' );
return;
}
// log started time.
self::save_summary( [ 'last_start_time' => time() ] );
// set time limit.
$max_time = (int) ini_get( 'max_execution_time' );
self::debug( 'ini_get max_execution_time=' . $max_time );
if ( 0 === $max_time ) {
$max_time = 300; // hardlimit.
} else {
$max_time -= 5;
}
if ( $max_time >= (int) $this->_crawler_conf['run_duration'] ) {
$max_time = (int) $this->_crawler_conf['run_duration'];
self::debug( 'Use run_duration setting as max_execution_time=' . $max_time );
// phpcs:ignore WordPress.PHP.IniSet.max_execution_time_Disallowed -- Required for crawler functionality.
} elseif ( ini_set( 'max_execution_time', $this->_crawler_conf['run_duration'] + 15 ) !== false ) {
$max_time = $this->_crawler_conf['run_duration'];
self::debug( 'ini_set max_execution_time=' . $max_time );
}
self::debug( 'final max_execution_time=' . $max_time );
$this->_max_run_time = $max_time + time();
// mark running.
$this->_prepare_running();
// run crawler.
$this->_do_running();
$this->_terminate_running();
}
/**
* Get server load.
*
* @since 5.5
* @return int Load or -1 if unsupported.
*/
public function get_server_load() {
if ( ! function_exists( 'sys_getloadavg' ) ) {
return -1;
}
$curload = sys_getloadavg();
$curload = (float) $curload[0];
self::debug( 'Server load: ' . $curload );
return $curload;
}
/**
* Adjust threads dynamically.
*
* @since 1.1.0
* @access private
* @return void
*/
private function _adjust_current_threads() {
$curload = $this->get_server_load();
if ( -1 === (int) $curload ) {
self::debug( 'set threads=0 due to func sys_getloadavg not exist!' );
$this->_cur_threads = 0;
return;
}
$curload /= (float) $this->_ncpu;
$crawler_threads = defined( 'LITESPEED_CRAWLER_THREADS' ) ? (int) constant( 'LITESPEED_CRAWLER_THREADS' ) : 3;
$load_limit = (float) $this->_crawler_conf['load_limit'];
$current_threads = (int) $this->_cur_threads;
if ( -1 === $current_threads ) {
// init.
if ( $curload > $load_limit ) {
$curthreads = 0;
} elseif ( $curload >= ( $load_limit - 1 ) ) {
$curthreads = 1;
} else {
$curthreads = (int) ( $load_limit - $curload );
if ( $curthreads > $crawler_threads ) {
$curthreads = $crawler_threads;
}
}
} else {
// adjust.
$curthreads = $current_threads;
if ( $curload >= ( $load_limit + 1 ) ) {
sleep( 5 ); // sleep 5 secs.
if ( $curthreads >= 1 ) {
--$curthreads;
}
} elseif ( $curload >= $load_limit ) {
--$curthreads;
} elseif ( ( $curload + 1 ) < $load_limit ) {
if ( $curthreads < $crawler_threads ) {
++$curthreads;
}
}
}
$this->_cur_threads = (int) $curthreads;
$this->_cur_thread_time = time();
}
/**
* Mark running status.
*
* @since 1.1.0
* @access private
* @return void
*/
private function _prepare_running() {
$this->_summary['is_running'] = time();
$this->_summary['done'] = 0; // reset done status.
$this->_summary['last_status'] = 'prepare running';
$this->_summary['last_crawled'] = 0;
// Current crawler starttime mark.
if ( 0 === (int) $this->_summary['last_pos'] ) {
$this->_summary['curr_crawler_beginning_time'] = time();
}
if ( 0 === (int) $this->_summary['curr_crawler'] && 0 === (int) $this->_summary['last_pos'] ) {
$this->_summary['this_full_beginning_time'] = time();
$this->_summary['list_size'] = $this->cls( 'Crawler_Map' )->count_map();
}
if ( 'end' === $this->_summary['end_reason'] && 0 === (int) $this->_summary['last_pos'] ) {
$this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ] = [];
}
self::save_summary();
}
/**
* Take over lane.
*
* @since 6.1
* @return void
*/
private function _take_over_lane() {
self::debug( 'Take over lane as lane is free: ' . $this->json_local_path() . '.pid' );
File::save( $this->json_local_path() . '.pid', LITESPEED_LANE_HASH );
}
/**
* Update lane file mtime.
*
* @since 6.1
* @return void
*/
private function _touch_lane() {
// phpcs:ignore WordPress.WP.AlternativeFunctions.file_system_operations_touch
touch( $this->json_local_path() . '.pid' );
}
/**
* Release lane file.
*
* @since 6.1
* @return void
*/
public function Release_lane() {
$lane_file = $this->json_local_path() . '.pid';
if ( ! file_exists( $lane_file ) ) {
return;
}
self::debug( 'Release lane' );
// phpcs:ignore WordPress.WP.AlternativeFunctions.unlink_unlink
unlink( $lane_file );
}
/**
* Check if lane is used by other crawlers.
*
* @since 6.1
* @param bool $strict_mode Strict check that file must exist.
* @return bool True if valid lane.
*/
private function _check_valid_lane( $strict_mode = false ) {
$lane_file = $this->json_local_path() . '.pid';
if ( $strict_mode ) {
if ( ! file_exists( $lane_file ) ) {
self::debug( 'lane file not existed, strict mode is false [file] ' . $lane_file );
return false;
}
}
$pid = File::read( $lane_file );
if ( $pid && LITESPEED_LANE_HASH !== $pid ) {
// If lane file is older than 1h, ignore.
if ( ( time() - filemtime( $lane_file ) ) > 3600 ) {
self::debug( 'Lane file is older than 1h, releasing lane' );
$this->Release_lane();
return true;
}
return false;
}
return true;
}
/**
* Test port for simulator.
*
* @since 7.0
* @access private
* @return bool true if success and can continue crawling, false otherwise.
*/
private function _test_port() {
if ( empty( $this->_server_ip ) ) {
if ( empty( $this->_crawlers[ $this->_summary['curr_crawler'] ]['uid'] ) ) {
self::debug( 'Bypass test port as Server IP is not set' );
return true;
}
self::debug( '❌ Server IP not set' );
return false;
}
if ( defined( 'LITESPEED_CRAWLER_LOCAL_PORT' ) ) {
self::debug( '✅ LITESPEED_CRAWLER_LOCAL_PORT already defined' );
return true;
}
// Don't repeat testing in 120s.
if ( ! empty( $this->_summary['test_port_tts'] ) && ( time() - (int) $this->_summary['test_port_tts'] ) < 120 ) {
if ( ! empty( $this->_summary['test_port'] ) ) {
self::debug( '✅ Use tested local port: ' . $this->_summary['test_port'] );
define( 'LITESPEED_CRAWLER_LOCAL_PORT', (int) $this->_summary['test_port'] );
return true;
}
return false;
}
$this->_summary['test_port_tts'] = time();
self::save_summary();
$options = $this->_get_curl_options();
$home = home_url();
File::save( LITESPEED_STATIC_DIR . '/crawler/test_port.html', $home, true );
$url = LITESPEED_STATIC_URL . '/crawler/test_port.html';
$parsed_url = wp_parse_url( $url );
if ( empty( $parsed_url['host'] ) ) {
self::debug( '❌ Test port failed, invalid URL: ' . $url );
return false;
}
$resolved = $parsed_url['host'] . ':443:' . $this->_server_ip;
$options[ CURLOPT_RESOLVE ] = [ $resolved ];
$options[ CURLOPT_DNS_USE_GLOBAL_CACHE ] = false;
$options[ CURLOPT_HEADER ] = false;
self::debug( 'Test local 443 port for ' . $resolved );
// cURL is intentionally used for speed; suppress sniffs in this method.
// phpcs:disable WordPress.WP.AlternativeFunctions
$ch = curl_init();
curl_setopt_array( $ch, $options );
curl_setopt( $ch, CURLOPT_URL, $url );
$result = curl_exec( $ch );
$test_result = false;
if ( curl_errno( $ch ) || $result !== $home ) {
if ( curl_errno( $ch ) ) {
self::debug( '❌ Test port curl error: [errNo] ' . curl_errno( $ch ) . ' [err] ' . curl_error( $ch ) );
} elseif ( $result !== $home ) {
self::debug( '❌ Test port response is wrong: ' . $result );
}
self::debug( '❌ Test local 443 port failed, try port 80' );
// Try port 80.
$resolved = $parsed_url['host'] . ':80:' . $this->_server_ip;
$options[ CURLOPT_RESOLVE ] = [ $resolved ];
$url = str_replace( 'https://', 'http://', $url );
if ( empty( $options[ CURLOPT_HTTPHEADER ] ) || ! in_array( 'X-Forwarded-Proto: https', $options[ CURLOPT_HTTPHEADER ], true ) ) {
$options[ CURLOPT_HTTPHEADER ][] = 'X-Forwarded-Proto: https';
}
$ch = curl_init();
curl_setopt_array( $ch, $options );
curl_setopt( $ch, CURLOPT_URL, $url );
$result = curl_exec( $ch );
if ( curl_errno( $ch ) ) {
self::debug( '❌ Test port curl error: [errNo] ' . curl_errno( $ch ) . ' [err] ' . curl_error( $ch ) );
} elseif ( $result !== $home ) {
self::debug( '❌ Test port response is wrong: ' . $result );
} else {
self::debug( '✅ Test local 80 port successfully' );
define( 'LITESPEED_CRAWLER_LOCAL_PORT', 80 );
$this->_summary['test_port'] = 80;
$test_result = true;
}
} else {
self::debug( '✅ Tested local 443 port successfully' );
define( 'LITESPEED_CRAWLER_LOCAL_PORT', 443 );
$this->_summary['test_port'] = 443;
$test_result = true;
}
self::save_summary();
curl_close( $ch );
// phpcs:enable
return $test_result;
}
/**
* Run crawler.
*
* @since 1.1.0
* @access private
* @return void
* @throws \Exception When lane becomes invalid during run.
*/
private function _do_running() {
$options = $this->_get_curl_options( true );
// If is role simulator and not defined local port, check port once.
$test_result = $this->_test_port();
if ( ! $test_result ) {
$this->_end_reason = 'port_test_failed';
self::debug( '❌ Test port failed, crawler stopped.' );
return;
}
while ( true ) {
$url_chunks = $this->cls( 'Crawler_Map' )->list_map( self::CHUNKS, $this->_summary['last_pos'] );
if ( empty( $url_chunks ) ) {
break;
}
$url_chunks = array_chunk( $url_chunks, (int) $this->_cur_threads );
foreach ( $url_chunks as $rows ) {
if ( ! $this->_check_valid_lane( true ) ) {
$this->_end_reason = 'lane_invalid';
self::debug( '🛑 The crawler lane is used by newer crawler.' );
throw new \Exception( 'invalid crawler lane' );
}
// Update time.
$this->_touch_lane();
// multi curl.
$rets = $this->_multi_request( $rows, $options );
// check result headers.
foreach ( $rows as $row ) {
if ( empty( $rets[ $row['id'] ] ) ) {
continue;
}
if ( 428 === (int) $rets[ $row['id'] ]['code'] ) {
// HTTP/1.1 428 Precondition Required (need to test)
$this->_end_reason = 'crawler_disabled';
self::debug( 'crawler_disabled' );
return;
}
$status = $this->_status_parse( $rets[ $row['id'] ]['header'], $rets[ $row['id'] ]['code'], $row['url'] ); // B or H or M or N(nocache).
self::debug( '[status] ' . $this->_status2title( $status ) . "\t\t [url] " . $row['url'] );
$this->_map_status_list[ $status ][ $row['id'] ] = [
'url' => $row['url'],
'code' => (int) $rets[ $row['id'] ]['code'], // 201 or 200 or 404.
];
if ( empty( $this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ][ $status ] ) ) {
$this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ][ $status ] = 0;
}
++$this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ][ $status ];
}
// update offset position.
$_time = time();
$this->_summary['last_count'] = count( $rows );
$this->_summary['last_pos'] += $this->_summary['last_count'];
$this->_summary['last_crawled'] += $this->_summary['last_count'];
$this->_summary['last_update_time'] = $_time;
$this->_summary['last_status'] = 'updated position';
// check duration.
if ( $this->_summary['last_update_time'] > $this->_max_run_time ) {
$this->_end_reason = 'stopped_maxtime';
self::debug( 'Terminated due to maxtime' );
return;
}
// make sure at least each 10s save meta & map status once.
if ( $_time - $this->_summary['meta_save_time'] > 10 ) {
$this->_map_status_list = $this->cls( 'Crawler_Map' )->save_map_status( $this->_map_status_list, $this->_summary['curr_crawler'] );
self::save_summary();
}
// check if need to reset pos each 5s.
if ( $_time > $this->_summary['pos_reset_check'] ) {
$this->_summary['pos_reset_check'] = $_time + 5;
if ( file_exists( $this->_resetfile ) && unlink( $this->_resetfile ) ) { // phpcs:ignore WordPress.WP.AlternativeFunctions.unlink_unlink
self::debug( 'Terminated due to reset file' );
$this->_summary['last_pos'] = 0;
$this->_summary['curr_crawler'] = 0;
$this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ] = [];
// reset done status.
$this->_summary['done'] = 0;
$this->_summary['this_full_beginning_time'] = 0;
$this->_end_reason = 'stopped_reset';
return;
}
}
// check loads.
if ( ( $this->_summary['last_update_time'] - $this->_cur_thread_time ) > 60 ) {
$this->_adjust_current_threads();
if ( 0 === (int) $this->_cur_threads ) {
$this->_end_reason = 'stopped_highload';
self::debug( '🛑 Terminated due to highload' );
return;
}
}
$this->_summary['last_status'] = 'sleeping ' . (int) $this->_crawler_conf['run_delay'] . 'ms';
usleep( (int) $this->_crawler_conf['run_delay'] );
}
}
// All URLs are done for current crawler.
$this->_end_reason = 'end';
$this->_summary['crawler_stats'][ $this->_summary['curr_crawler'] ]['W'] = 0;
self::debug( 'Crawler #' . $this->_summary['curr_crawler'] . ' touched end' );
}
/**
* If need to resolve DNS or not.
*
* @since 7.3.0.1
* @return bool
*/
private function _should_force_resolve_dns() {
if ( ! empty( $this->_server_ip ) ) {
return true;
}
if ( ! empty( $this->_crawler_conf['cookies'] ) && ! empty( $this->_crawler_conf['cookies']['litespeed_hash'] ) ) {
return true;
}
return false;
}
/**
* Send multi curl requests.
* If res=B/N, bypass request and won't return.
*
* @since 1.1.0
* @access private
*
* @param array<int,array<string,mixed>> $rows Rows to crawl.
* @param array $options cURL options.
* @return array<int,array{header:string,code:int}>
*/
private function _multi_request( $rows, $options ) {
if ( ! function_exists( 'curl_multi_init' ) ) {
exit( 'curl_multi_init disabled' );
}
// phpcs:disable WordPress.WP.AlternativeFunctions
$mh = curl_multi_init();
$crawler_drop_domain = defined( 'LITESPEED_CRAWLER_DROP_DOMAIN' ) ? (bool) constant( 'LITESPEED_CRAWLER_DROP_DOMAIN' ) : false;
$curls = [];
foreach ( $rows as $row ) {
if ( self::STATUS_BLACKLIST === substr( $row['res'], $this->_summary['curr_crawler'], 1 ) ) {
continue;
}
if ( self::STATUS_NOCACHE === substr( $row['res'], $this->_summary['curr_crawler'], 1 ) ) {
continue;
}
if (!function_exists('curl_init')) {
exit('curl_init disabled');
}
$curls[$row['id']] = curl_init();
// Append URL.
$url = $row['url'];
if ( $crawler_drop_domain ) {
$url = $this->_crawler_conf['base'] . $row['url'];
}
// IP resolve.
if ( $this->_should_force_resolve_dns() ) {
$parsed_url = wp_parse_url( $url );
if ( ! empty( $parsed_url['host'] ) ) {
$dom = $parsed_url['host'];
$port = defined( 'LITESPEED_CRAWLER_LOCAL_PORT' ) ? (int) LITESPEED_CRAWLER_LOCAL_PORT : 443;
$resolved = $dom . ':' . $port . ':' . $this->_server_ip;
$options[ CURLOPT_RESOLVE ] = [ $resolved ];
$options[ CURLOPT_DNS_USE_GLOBAL_CACHE ] = false;
if ( 80 === $port ) {
$url = str_replace( 'https://', 'http://', $url );
if ( empty( $options[ CURLOPT_HTTPHEADER ] ) || ! in_array( 'X-Forwarded-Proto: https', $options[ CURLOPT_HTTPHEADER ], true ) ) {
$options[ CURLOPT_HTTPHEADER ][] = 'X-Forwarded-Proto: https';
}
}
self::debug( 'Resolved DNS for ' . $resolved );
}
}
curl_setopt( $curls[ $row['id'] ], CURLOPT_URL, $url );
self::debug( 'Crawling [url] ' . $url . ( $url === $row['url'] ? '' : ' [ori] ' . $row['url'] ) );
curl_setopt_array( $curls[ $row['id'] ], $options );
curl_multi_add_handle( $mh, $curls[ $row['id'] ] );
}
// execute curl.
if ( $curls ) {
do {
$status = curl_multi_exec( $mh, $active );
if ( $active ) {
curl_multi_select( $mh );
}
} while ( $active && CURLM_OK === $status );
}
// curl done.
$ret = [];
foreach ( $rows as $row ) {
if ( self::STATUS_BLACKLIST === substr( $row['res'], $this->_summary['curr_crawler'], 1 ) ) {
continue;
}
if ( self::STATUS_NOCACHE === substr( $row['res'], $this->_summary['curr_crawler'], 1 ) ) {
continue;
}
$ch = $curls[ $row['id'] ];
// Parse header.
$header_size = curl_getinfo( $ch, CURLINFO_HEADER_SIZE );
$content = curl_multi_getcontent( $ch );
$header = substr( $content, 0, $header_size );
$ret[ $row['id'] ] = [
'header' => $header,
'code' => (int) curl_getinfo( $ch, CURLINFO_HTTP_CODE ),
];
curl_multi_remove_handle( $mh, $ch );
curl_close( $ch );
}
curl_multi_close( $mh );
// phpcs:enable
return $ret;
}
/**
* Translate the status to title.
*
* @since 6.0
* @param string $status Status char.
* @return string Human title.
*/
private function _status2title( $status ) {
if ( self::STATUS_HIT === $status ) {
return '✅ Hit';
}
if ( self::STATUS_MISS === $status ) {
return '😊 Miss';
}
if ( self::STATUS_BLACKLIST === $status ) {
return '😅 Blacklisted';
}
if ( self::STATUS_NOCACHE === $status ) {
return '😅 Blacklisted';
}
return '🛸 Unknown';
}
/**
* Check returned curl header to find if cached or not.
*
* @since 2.0
* @access private
*
* @param string $header Response header.
* @param int $code HTTP code.
* @param string $url URL.
* @return string One of status chars.
*/
private function _status_parse( $header, $code, $url ) {
if ( 201 === (int) $code ) {
return self::STATUS_HIT;
}
if ( false !== stripos( $header, 'X-Litespeed-Cache-Control: no-cache' ) ) {
// If is from DIVI, taken as miss.
if ( defined( 'LITESPEED_CRAWLER_IGNORE_NONCACHEABLE' ) && constant( 'LITESPEED_CRAWLER_IGNORE_NONCACHEABLE' ) ) {
return self::STATUS_MISS;
}
// If blacklist is disabled.
if ( ( defined( 'LITESPEED_CRAWLER_DISABLE_BLOCKLIST' ) && constant( 'LITESPEED_CRAWLER_DISABLE_BLOCKLIST' ) ) || apply_filters( 'litespeed_crawler_disable_blocklist', false, $url ) ) {
return self::STATUS_MISS;
}
return self::STATUS_NOCACHE; // Blacklist.
}
$_cache_headers = [ 'x-litespeed-cache', 'x-qc-cache', 'x-lsadc-cache' ];
foreach ( $_cache_headers as $_header ) {
if ( false !== stripos( $header, $_header ) ) {
if ( false !== stripos( $header, $_header . ': bkn' ) ) {
return self::STATUS_HIT; // Hit.
}
if ( false !== stripos( $header, $_header . ': miss' ) ) {
return self::STATUS_MISS; // Miss.
}
return self::STATUS_HIT; // Hit.
}
}
// If blacklist is disabled.
if ( ( defined( 'LITESPEED_CRAWLER_DISABLE_BLOCKLIST' ) && constant( 'LITESPEED_CRAWLER_DISABLE_BLOCKLIST' ) ) || apply_filters( 'litespeed_crawler_disable_blocklist', false, $url ) ) {
return self::STATUS_MISS;
}
return self::STATUS_BLACKLIST; // Blacklist.
}
/**
* Get curl options.
*
* @since 1.1.0
* @access private
*
* @param bool $crawler_only Whether crawler-only UA.
* @return array
*/
private function _get_curl_options( $crawler_only = false ) {
$crawler_timeout = defined( 'LITESPEED_CRAWLER_TIMEOUT' ) ? (int) constant( 'LITESPEED_CRAWLER_TIMEOUT' ) : 30;
$options = [
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HEADER => true,
CURLOPT_CUSTOMREQUEST => 'GET',
CURLOPT_FOLLOWLOCATION => false,
CURLOPT_ENCODING => 'gzip',
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_TIMEOUT => $crawler_timeout, // Larger timeout to avoid incorrect blacklist addition #900171.
CURLOPT_SSL_VERIFYHOST => 0,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_NOBODY => false,
CURLOPT_HTTPHEADER => $this->_crawler_conf['headers'],
];
$options[ CURLOPT_HTTPHEADER ][] = 'Cache-Control: max-age=0';
$options[ CURLOPT_HTTP_VERSION ] = CURL_HTTP_VERSION_1_1;
// if is walker
// $options[ CURLOPT_FRESH_CONNECT ] = true;
// Referer.
if ( isset( $_SERVER['HTTP_HOST'], $_SERVER['REQUEST_URI'] ) ) {
$host = sanitize_text_field( wp_unslash( $_SERVER['HTTP_HOST'] ) );
$uri = sanitize_text_field( wp_unslash( $_SERVER['REQUEST_URI'] ) );
$options[ CURLOPT_REFERER ] = 'http://' . $host . $uri;
}
// User Agent.
if ( $crawler_only ) {
if ( 0 !== strpos( (string) $this->_crawler_conf['ua'], self::FAST_USER_AGENT ) ) {
$this->_crawler_conf['ua'] = self::FAST_USER_AGENT . ' ' . (string) $this->_crawler_conf['ua'];
}
}
$options[ CURLOPT_USERAGENT ] = (string) $this->_crawler_conf['ua'];
// Cookies.
$cookies = [];
foreach ( $this->_crawler_conf['cookies'] as $k => $v ) {
if ( ! $v ) {
continue;
}
$cookies[] = $k . '=' . rawurlencode( $v );
}
if ( $cookies ) {
$options[ CURLOPT_COOKIE ] = implode( '; ', $cookies );
}
return $options;
}
/**
* Self curl to get HTML content.
*
* @since 3.3
*
* @param string $url URL.
* @param string $ua User agent.
* @param int|false $uid Optional user ID for simulation.
* @param string|false $accept Optional Accept header value.
* @return string|false HTML on success, false on failure.
*/
public function self_curl( $url, $ua, $uid = false, $accept = false ) {
$this->_crawler_conf['base'] = site_url();
$this->_crawler_conf['ua'] = $ua;
if ( $accept ) {
$this->_crawler_conf['headers'] = [ 'Accept: ' . $accept ];
}
$options = $this->_get_curl_options();
if ( $uid ) {
$this->_crawler_conf['cookies']['litespeed_flash_hash'] = Router::cls()->get_flash_hash( $uid );
$parsed_url = wp_parse_url( $url );
if ( ! empty( $parsed_url['host'] ) ) {
$dom = $parsed_url['host'];
$port = defined( 'LITESPEED_CRAWLER_LOCAL_PORT' ) ? (int) LITESPEED_CRAWLER_LOCAL_PORT : 443;
$resolved = $dom . ':' . $port . ':' . $this->_server_ip;
$options[ CURLOPT_RESOLVE ] = [ $resolved ];
$options[ CURLOPT_DNS_USE_GLOBAL_CACHE ] = false;
$options[ CURLOPT_PORT ] = $port;
self::debug( 'Resolved DNS for ' . $resolved );
}
}
$options[ CURLOPT_HEADER ] = false;
$options[ CURLOPT_FOLLOWLOCATION ] = true;
// phpcs:disable WordPress.WP.AlternativeFunctions
$ch = curl_init();
curl_setopt_array( $ch, $options );
curl_setopt( $ch, CURLOPT_URL, $url );
$result = curl_exec( $ch );
$code = (int) curl_getinfo( $ch, CURLINFO_HTTP_CODE );
curl_close( $ch );
// phpcs:enable
if ( 200 !== $code ) {
self::debug( '❌ Response code is not 200 in self_curl() [code] ' . $code );
return false;
}
return $result;
}
/**
* Terminate crawling.
*
* @since 1.1.0
* @access private
* @return void
*/
private function _terminate_running() {
$this->_map_status_list = $this->cls( 'Crawler_Map' )->save_map_status( $this->_map_status_list, $this->_summary['curr_crawler'] );
if ( 'end' === $this->_end_reason ) {
$this->_summary['curr_crawler'] = (int) $this->_summary['curr_crawler'] + 1; // Jump to next crawler.
$this->_summary['last_pos'] = 0; // reset last position.
$this->_summary['last_crawler_total_cost'] = time() - (int) $this->_summary['curr_crawler_beginning_time'];
$count_crawlers = count( $this->list_crawlers() );
if ( $this->_summary['curr_crawler'] >= $count_crawlers ) {
self::debug( '_terminate_running Touched end, whole crawled. Reload crawler!' );
$this->_summary['curr_crawler'] = 0;
$this->_summary['done'] = 'touchedEnd'; // log done status.
$this->_summary['last_full_time_cost'] = time() - (int) $this->_summary['this_full_beginning_time'];
}
}
$this->_summary['last_status'] = 'stopped';
$this->_summary['is_running'] = 0;
$this->_summary['end_reason'] = $this->_end_reason;
self::save_summary();
}
/**
* List all crawlers ( tagA => [ valueA => titleA, ... ] ... ).
*
* @since 1.9.1
* @access public
* @return array<int,array<string,mixed>>
*/
public function list_crawlers() {
if ( $this->_crawlers ) {
return $this->_crawlers;
}
$crawler_factors = [];
// Add default Guest crawler.
$crawler_factors['uid'] = [ 0 => __( 'Guest', 'litespeed-cache' ) ];
// WebP on/off.
if ( $this->conf( Base::O_IMG_OPTM_WEBP ) ) {
$crawler_factors['webp'] = [ 1 => $this->cls( 'Media' )->next_gen_image_title() ];
if ( apply_filters( 'litespeed_crawler_webp', false ) ) {
$crawler_factors['webp'][0] = '';
}
}
// Guest Mode on/off.
if ( $this->conf( Base::O_GUEST ) ) {
$vary_name = $this->cls( 'Vary' )->get_vary_name();
$vary_val = 'guest_mode:1';
if ( ! defined( 'LSCWP_LOG' ) ) {
$vary_val = md5( $this->conf( Base::HASH ) . $vary_val );
}
$crawler_factors[ 'cookie:' . $vary_name ] = [
$vary_val => '',
'_null' => '<font data-balloon-pos="up" aria-label="Guest Mode">👒</font>',
];
}
// Mobile crawler.
if ( $this->conf( Base::O_CACHE_MOBILE ) ) {
$crawler_factors['mobile'] = [
1 => '<font data-balloon-pos="up" aria-label="Mobile">📱</font>',
0 => '',
];
}
// Get roles set.
foreach ( $this->conf( Base::O_CRAWLER_ROLES ) as $v ) {
$role_title = '';
$udata = get_userdata( $v );
if ( isset( $udata->roles ) && is_array( $udata->roles ) ) {
$tmp = array_values( $udata->roles );
$role_title = array_shift( $tmp );
}
if ( ! $role_title ) {
continue;
}
$crawler_factors['uid'][ $v ] = ucfirst( $role_title );
}
// Cookie crawler.
foreach ( $this->conf( Base::O_CRAWLER_COOKIES ) as $v ) {
if ( empty( $v['name'] ) ) {
continue;
}
$this_cookie_key = 'cookie:' . $v['name'];
$crawler_factors[ $this_cookie_key ] = [];
foreach ( $v['vals'] as $v2 ) {
$crawler_factors[ $this_cookie_key ][ $v2 ] =
( '_null' === $v2 ? '' : '<font data-balloon-pos="up" aria-label="Cookie">🍪</font>' . esc_html( $v['name'] ) . '=' . esc_html( $v2 ) );
}
}
// Crossing generate the crawler list.
$this->_crawlers = $this->_recursive_build_crawler( $crawler_factors );
return $this->_crawlers;
}
/**
* Build a crawler list recursively.
*
* @since 2.8
* @access private
*
* @param array<string,array> $crawler_factors Factors.
* @param array $group Current group.
* @param int $i Factor index.
* @return array<int,array>
*/
private function _recursive_build_crawler( $crawler_factors, $group = [], $i = 0 ) {
$current_factor_keys = array_keys( $crawler_factors );
$current_factor = $current_factor_keys[ $i ];
$if_touch_end = ( $i + 1 ) >= count( $crawler_factors );
$final_list = [];
foreach ( $crawler_factors[ $current_factor ] as $k => $v ) {
$item = $group; // Don't alter $group bcos of loop usage.
$item['title'] = ! empty( $group['title'] ) ? $group['title'] : '';
if ( $v ) {
if ( $item['title'] ) {
$item['title'] .= ' - ';
}
$item['title'] .= $v;
}
$item[ $current_factor ] = $k;
if ( $if_touch_end ) {
$final_list[] = $item;
} else {
// Inception: next layer.
$final_list = array_merge( $final_list, $this->_recursive_build_crawler( $crawler_factors, $item, $i + 1 ) );
}
}
return $final_list;
}
/**
* Return crawler meta file local path.
*
* @since 6.1
* @access public
* @return string
*/
public function json_local_path() {
return LITESPEED_STATIC_DIR . '/crawler/' . $this->_sitemeta;
}
/**
* Return crawler meta file URL.
*
* @since 1.1.0
* @access public
* @return string|false
*/
public function json_path() {
if ( ! file_exists( LITESPEED_STATIC_DIR . '/crawler/' . $this->_sitemeta ) ) {
return false;
}
return LITESPEED_STATIC_URL . '/crawler/' . $this->_sitemeta;
}
/**
* Create reset pos file.
*
* @since 1.1.0
* @access public
* @return void
*/
public function reset_pos() {
File::save( $this->_resetfile, time(), true );
self::save_summary( [ 'is_running' => 0 ] );
}
/**
* Display status based by matching crawlers order.
*
* @since 3.0
* @access public
*
* @param string $status_row Status string.
* @param string $reason_set Comma separated reasons.
* @return string HTML dots.
*/
public function display_status( $status_row, $reason_set ) {
if ( ! $status_row ) {
return '';
}
$_status_list = [
'-' => 'default',
self::STATUS_MISS => 'primary',
self::STATUS_HIT => 'success',
self::STATUS_BLACKLIST => 'danger',
self::STATUS_NOCACHE => 'warning',
];
$reason_set = explode( ',', $reason_set );
$status = '';
foreach ( str_split( $status_row ) as $k => $v ) {
$reason = isset( $reason_set[ $k ] ) ? $reason_set[ $k ] : '';
if ( 'Man' === $reason ) {
$reason = __( 'Manually added to blocklist', 'litespeed-cache' );
}
if ( 'Existed' === $reason ) {
$reason = __( 'Previously existed in blocklist', 'litespeed-cache' );
}
$reason_attr = $reason ? 'data-balloon-pos="up" aria-label="' . esc_attr( $reason ) . '"' : '';
$status .= '<i class="litespeed-dot litespeed-bg-' . esc_attr( $_status_list[ $v ] ) . '" ' . $reason_attr . '>' . ( $k + 1 ) . '</i>';
}
return $status;
}
/**
* Handle all request actions from main cls.
*
* @since 3.0
* @access public
* @return void
*/
public function handler() {
$type = Router::verify_type();
switch ( $type ) {
case self::TYPE_REFRESH_MAP:
$this->cls( 'Crawler_Map' )->gen( true );
break;
case self::TYPE_EMPTY:
$this->cls( 'Crawler_Map' )->empty_map();
break;
case self::TYPE_BLACKLIST_EMPTY:
$this->cls( 'Crawler_Map' )->blacklist_empty();
break;
case self::TYPE_BLACKLIST_DEL:
// phpcs:ignore WordPress.Security.NonceVerification.Recommended, WordPress.Security.ValidatedSanitizedInput.MissingUnslash, WordPress.Security.ValidatedSanitizedInput.InputNotSanitized
if (!empty($_GET['id'])) {
// phpcs:ignore WordPress.Security.NonceVerification.Recommended
$id = absint( wp_unslash( $_GET['id'] ) );
$this->cls( 'Crawler_Map' )->blacklist_del( $id );
}
break;
case self::TYPE_BLACKLIST_ADD:
// phpcs:ignore WordPress.Security.NonceVerification.Recommended, WordPress.Security.ValidatedSanitizedInput.MissingUnslash, WordPress.Security.ValidatedSanitizedInput.InputNotSanitized
if (!empty($_GET['id'])) {
// phpcs:ignore WordPress.Security.NonceVerification.Recommended
$id = absint( wp_unslash( $_GET['id'] ) );
$this->cls( 'Crawler_Map' )->blacklist_add( $id );
}
break;
case self::TYPE_START: // Handle the ajax request to proceed crawler manually by admin.
self::start_async();
break;
case self::TYPE_RESET:
$this->reset_pos();
break;
default:
break;
}
Admin::redirect();
}
}