@ -16,7 +16,7 @@ class Keyworlds extends CI_Controller {
public function index($kw_is_id = '') {
$this->load->library('Accesscheck');
$this->accesscheck->check_access(); // test--
$this->accesscheck->check_access();
$this->permission->is_admin();
//查询结构列表信息
@ -263,7 +263,6 @@ class Keyworlds extends CI_Controller {
'vac' => 'get_search_content_by_google',
'it' => 'get_search_content_by_google',
'ru' => 'get_search_content_by_google',
'wt' => 'get_search_content_by_baidu',
'tbt' => 'get_search_content_by_google',
'sht' => 'get_search_content_by_google',
'yz' => 'get_search_content_by_google',
@ -272,48 +271,59 @@ class Keyworlds extends CI_Controller {
'ct' => 'get_search_content_by_google',
'mct' => 'get_search_content_by_google',
'dct' => 'get_search_content_by_google',
'cits' => 'get_search_content_by_baidu',
'ah' => 'get_search_content_by_google'
);
$site_code = $this->config->item('site_code');
$method = $engines[$site_code];
if ($site_code == 'wt')
$site_url = 'www.iiiyooo.com';
if ($site_code == 'cits')
$site_url = 'www.guilincits.c';
$last_rank = $this->Keywordsrank_model->get_rank_list($kw_id, $top = 1);
if (isset($last_rank[0]->kwr_datetime) and ( time() - $last_rank[0]->kwr_datetime) < 300 ) {
$today = strtotime(date('Y-m-d'));
$client_flag = 'PC';
$rank_flag = 'rank';
$source_flag = 'source';
$last_rank = $this->Keywordsrank_model->get_rank_list($kw_id, $top = 1);
if (isset($last_rank[0]->kwr_datetime) and ( time() - $last_rank[0]->kwr_datetime) < 180 ) {
$data[] = array('name' => 'rank', 'value' => '查询过于频繁,请等待三分钟!');
echo json_encode($data);
return TRUE;
} else if (empty($last_rank[0]->kwr_mobile_rank) & &
intval($last_rank[0]->kwr_datetime) > intval($today)) {
$client_flag = 'm';
$rank_flag = 'rank_m';
$source_flag = 'source_m';
}
//获取排名详细情况,排名超过150的停止搜索
for ($i = 1; $i < = 15; $i++) {
$j = $i;
$rank = $this->$method($keyworlds, $site_url, $j, $site_code);
if ($rank['rank'] != 0) {
if ($rank['rank'] > 150)
$rank['rank'] = '151';
$rank = $this->$method($client_flag, $keyworlds, $site_url, $j, $site_code);
// 仅更新一个字段
if (intval($rank['rank']) == 0) {
$rank['rank'] = $last_rank[0]->kwr_rank;
$rank['source'] = $last_rank[0]->kwr_source;
} else {
$rank['rank_m'] = null;
$rank['source_m'] = null;
}
if ($rank[$rank_flag] != 0) {
if ($rank[$rank_flag] > 150)
$rank[$rank_flag] = '151';
break;
}elseif ($i == 15) {
$rank['rank'] = '151';
$rank[$rank_flag ] = '151';
break;
} elseif ($rank['rank'] == -2) {
} elseif ($rank[$rank_flag ] == -2) {
break;
}
}
//保存排名数据
if ($rank['rank' ] == '151') { //超过151的排名则不记录
if ($rank[$rank_flag ] == '151') { //超过151的排名则不记录
$this->Keywords_model->update_time($kw_id);
} elseif (isset($last_rank[0]->kwr_datetime) and $last_rank[0]->kwr_datetime > strtotime(date('Y-m-d'))) {
if ($rank['rank'] != -1 and $rank['rank' ] != -2 and $this->Keywordsrank_model->update_rank($last_rank[0]->kwr_id, $rank['rank'], $rank['rank_m'], $rank['source'], $rank['source_m'])) {
if ($rank[$rank_flag] != -1 and $rank[$rank_flag ] != -2 and $this->Keywordsrank_model->update_rank($last_rank[0]->kwr_id, $rank['rank'], $rank['rank_m'], $rank['source'], $rank['source_m'])) {
$this->Keywords_model->update_time($kw_id);
}
} else {
if ($rank['rank'] != -1 and $rank['rank' ] != -2 and $this->Keywordsrank_model->add_rank($kw_id, $rank['rank'], $rank['rank_m'], $rank['source'], $rank['source_m'], $rank['engines'])) {
if ($rank[$rank_flag] != -1 and $rank[$rank_flag ] != -2 and $this->Keywordsrank_model->add_rank($kw_id, $rank['rank'], $rank['rank_m'], $rank['source'], $rank['source_m'], $rank['engines'])) {
$this->Keywords_model->update_time($kw_id);
}
}
@ -324,17 +334,26 @@ class Keyworlds extends CI_Controller {
//自动爬取排名
public function get_auto_rank($engines = '', $fix_site_code = '') {
set_time_limit(60);
$rank = array();
$site_config = $this->config->item('site');
$engines = trim($engines);
//取出需要更新排名的关键词
$worldsList = $this->Keywords_model->get_keyworld_list(1, $engines, $fix_site_code);
//print_r($worldsList);die();
//如果没有需要更新的则返回
if (empty($worldsList)) {
echo 'all done';
return;
}
$today = strtotime(date('Y-m-d'));
$client_flag = "PC";
$rank_flag = 'rank';
$source_flag = 'source';
if (intval($worldsList[0]->kwr_datetime) >= intval($today) ) {
$client_flag = "m";
$rank_flag = 'rank_m';
$source_flag = 'source_m';
}
$last_rank = $this->Keywordsrank_model->get_rank_list($worldsList[0]->kw_id);
//获取排名并保存
@ -345,24 +364,30 @@ class Keyworlds extends CI_Controller {
for ($i = 1; $i < = 15; $i++) {
if ($engines == '' || $engines == 'en' || $engines == 'eu') {//没有输入搜索引擎的时候根据站点自动分配获取排名的搜索引擎
$rank = $this->get_search_content_by_google($v->kw_keyworlds, $site_url, $i, $site_code);
$rank = $this->get_search_content_by_google($client_flag,$ v->kw_keyworlds, $site_url, $i, $site_code);
} else if ($engines == 'yandex') {//俄语站的yangdex
$rank = $this->get_search_content_by_yandex($v->kw_keyworlds, $site_url, $i - 1);
$rank = $this->get_search_content_by_yandex($client_flag,$ v->kw_keyworlds, $site_url, $i - 1);
}
if ($rank['rank'] != 0) {
if ($rank['rank'] > 150)
$rank['rank'] = '151';
if (intval($rank['rank']) == 0) {
$rank['rank'] = $last_rank[0]->kwr_rank;
$rank['source'] = $last_rank[0]->kwr_source;
} else {
$rank['rank_m'] = null;
$rank['source_m'] = null;
}
if ($rank[$rank_flag] != 0) {
if ($rank[$rank_flag] > 150)
$rank[$rank_flag] = '151';
break;
}elseif ($i == 15) {
$rank['rank'] = '151';
} elseif ($rank['rank'] == -2) {
$rank[$rank_flag ] = '151';
} elseif ($rank[$rank_flag ] == -2) {
break;
}
}
if ($rank['rank' ] == '151') { //超过151的排名则不记录
if ($rank[$rank_flag ] == '151') { //超过151的排名则不记录
$this->Keywords_model->update_time($v->kw_id);
}else if ($rank['rank'] != -1 and $rank['rank' ] != -2 and $this->Keywordsrank_model->add_rank($v->kw_id, $rank['rank'], $rank['rank_m'], $rank['source'], $rank['source_m'], $rank['engines'])) {
}else if ($rank[$rank_flag] != -1 and $rank[$rank_flag ] != -2 and $this->Keywordsrank_model->add_rank($v->kw_id, $rank['rank'], $rank['rank_m'], $rank['source'], $rank['source_m'], $rank['engines'])) {
$this->load->model('InfoSMS_model');
$last_rank = isset($last_rank[0]) ? $last_rank[0]->kwr_rank : $rank['rank'];
$this->InfoSMS_model->readed_for_info($v->kw_id, 'keywords_rank');
@ -375,7 +400,8 @@ class Keyworlds extends CI_Controller {
}
//计算关键词在google搜索引擎的排名
public function get_search_content_by_google($keyworlds, $site_url, $page, $site_code) {
public function get_search_content_by_google($client_flag,$keyworlds, $site_url, $page, $site_code) {
$site_url .= "/";
$rank['rank'] = 0;
$rank['source'] = $site_url;
$rank['rank_m'] = 0;
@ -410,23 +436,31 @@ class Keyworlds extends CI_Controller {
//抓取google搜索结果
$url = "https://www.$google_url/search?q=$search_worlds&start=$start&num=50&hl=$language";
$response = $this->curl($url);
$match_fun = '/< h3 class = "r" > (.*)< \/h3>/smUi';
$rank_flag = 'rank';
$source_flag = 'source';
if ($client_flag == 'm') {
$match_fun = '/< a class = "_Olt _bCp" ( . * ) < div / smUi ' ;
$rank_flag = 'rank_m';
$source_flag = 'source_m';
}
$response = $this->curl($url,$client_flag);
if ($response) {
//分析google搜索结果, 计算排名
if (false !== stripos($response, '< h1 > 302 Moved< / h1 > ')) {
$rank['rank'] = -2;
} elseif (false !== stripos($response, $site_url)) {
preg_match_all('/< h3 class = "r" > (.*)< \/h3>/smUi', $response, $store);
preg_match_all($match_fun , $response, $store);
foreach ($store[1] as $k => $v) {
$ismatch = stripos($v, $site_url);
if (false !== $ismatch) {
//计算排名
$rank['rank' ] = $start + $k + 1;
$rank[$rank_flag ] = $start + $k + 1;
//抓取关键词来源
$end = strpos($v, '"', $ismatch);
$source = substr($v, $ismatch, $end);
$source = explode('"', $source);
$rank['source' ] = $source[0];
$rank[$source_flag ] = $source[0];
break;
}
}
@ -434,43 +468,20 @@ class Keyworlds extends CI_Controller {
// log_message('error','google search response not found CH');
}
} else {
$rank['rank'] = -1;
}
// 移动端
$response_m = $this->curl($url,"m");
if ($response_m) {
if (false !== stripos($response_m, '< h1 > 302 Moved< / h1 > ')) {
$rank['rank_m'] = -2;
} elseif (false !== stripos($response_m, $site_url)) {
preg_match_all('/< a class = "_Olt _bCp" ( . * ) < div / smUi ' , $ response_m , $ store_m ) ;
foreach ($store_m[1] as $km => $vm) {
$ismatch_m = stripos($vm, $site_url);
if (false !== $ismatch_m) {
//计算排名
$rank['rank_m'] = $start + $km + 1;
//抓取关键词来源
$end_m = strpos($vm, '"', $ismatch_m);
$source_m = substr($vm, $ismatch_m, $end_m);
$source_m = explode('"', $source_m);
$rank['source_m'] = $source_m[0];
break;
}
}
}
} else {
$rank['rank_m'] = -1;
} // ---- 移动端 end ----
$rank[$rank_flag] = -1;
}
return $rank;
}
public function test() {
$ranks = $this->get_search_content_by_yandex('Путеводитель по городу гуйлинь', 'www.chinahighlights.ru');
// $ranks = $this->get_search_content_by_yandex('Путеводитель по городу гуйлинь', 'www.chinahighlights.ru');
// $ranks = $this->get_search_content_by_google('PC','travel china', 'https://www.chinahighlights.com',1,"en");
var_dump($ranks);
}
//计算关键词在yandex搜索引擎的排名
public function get_search_content_by_yandex($keyworlds, $site_url, $page = 0, $sitecode = false) {
public function get_search_content_by_yandex($client_flag,$keyworlds, $site_url, $page = 0, $sitecode = false) {
$site_url .= "/";
$rank['rank'] = 0;
$rank['rank_m'] = 0;
$rank['source'] = $site_url;
@ -485,58 +496,41 @@ class Keyworlds extends CI_Controller {
$search_worlds = urlencode($keyworlds);
//抓取搜索结果数据
$url = "https://www.yandex.ru/yandsearch?p=$page&text=$search_worlds&lr=21436";
$refer = "https://www.yandex.ru/";
$response = $this->curl($url,'PC',$refer);
$url = "https://www.yandex.ru/yandsearch?p=$page&text=$search_worlds&lr=21436";
$refer = "https://www.yandex.ru/";
$match_fun = '/< a class = \"link link_theme_normal organic__url link_cropped_no i-bem \ " ( . * ) class = /smUi';
$rank_flag = 'rank';
$source_flag = 'source';
if ($client_flag == 'm') {
$match_fun = '/< a class = "link link_theme_normal organic__url i-bem" ( . * ) class = /smUi';
$rank_flag = 'rank_m';
$source_flag = 'source_m';
}
$response = $this->curl($url,$client_flag,$refer);
if ($response) {
// log_message('error','yandex 1');
//分析YANDEX搜索结果, 计算排名
// 分析YANDEX搜索结果, 计算排名
if (false !== stripos($response, '< h1 > 302 Found< / h1 > ') ||
false !== stripos($response, 'капчей') || false !== stripos($response, 'captcha')) {
$rank['rank'] = -2;
} elseif (false !== stripos($response, $site_url)) {
preg_match_all("/< a class = \"link link_theme_normal organic__url link_cropped_no i-bem \ " ( . * ) class = /smUi", $ response , $ store ) ;
preg_match_all($match_fun, $response, $store);
foreach ($store[1] as $k => $v) {
$start_s = stripos($v, $site_url);
if (false !== $start_s) {
$rank['rank' ] = $startrank + $k + 1;
$rank[$rank_flag ] = $startrank + $k + 1;
$end_s = stripos($v, '" ', $start_s);
$source = substr($v, $start_s, $end_s);
$sources = explode('" ', $source);
$rank['source' ] = $sources[0];
$rank[$source_flag ] = $sources[0];
break;
}
}
} else {
// log_message('error','yandex not found CH');
}
} else {
$rank['rank' ] = -1;
$rank[$rank_flag ] = -1;
}
// 移动端
$response_m = $this->curl($url,"m",$refer);
if ($response_m) {
if (false !== stripos($response_m, '< h1 > 302 Found< / h1 > ') ||
false !== stripos($response, 'капчей') || false !== stripos($response, 'captcha')) {
$rank['rank_m'] = -2;
} elseif (false !== stripos($response_m, $site_url)) {
preg_match_all('/< a class = "link link_theme_normal organic__url i-bem" ( . * ) class = /smUi', $ response_m , $ store_m ) ;
foreach ($store_m[1] as $km => $vm) {
$ismatch_m = stripos($vm, $site_url);
if (false !== $ismatch_m) {
//计算排名
$rank['rank_m'] = $startrank + $km + 1;
//抓取关键词来源
$end_m = strpos($vm, '"', $ismatch_m);
$source_m = substr($vm, $ismatch_m, $end_m);
$source_m = explode('"', $source_m);
$rank['source_m'] = $source_m[0];
break;
}
}
}
} else {
$rank['rank_m'] = -1;
} // ---- 移动端 end ----
return $rank;
}
@ -577,70 +571,6 @@ class Keyworlds extends CI_Controller {
}
}
//收录查询结果
public function check_embody() {
$this->load->model('InfoMetas_model');
$this->load->model('InfoSMS_model');
//获取未收录的数据
$today_date = date('Y-m-d', time() - 3 * 24 * 60 * 60);
$embody = $this->InfoMetas_model->get_unembody_content($today_date);
if (empty($embody)) {
return false;
}
//页面url
$embody = $embody[0];
if ($embody->ic_url == '')
return false;
$site = $this->config->item('site');
$site_url = $site[$embody->ic_sitecode]['site_url'];
$site_url = $site_url . $embody->ic_url;
//收录查询搜索url
$search_worlds = 'site:' . $site_url;
$search_worlds;
$google_url = 'google.com';
$language = 'en';
$language_array = array('gm' => 'de', 'vc' => 'fr', 'vac' => 'es', 'ru' => 'ru', 'it' => 'it', 'jp' => 'jp');
if (!empty($language_array[$embody->ic_sitecode]))
$language = $language_array[$embody->ic_sitecode];
//抓取google搜索结果
$this->load->library('Snoopy');
$snoopy = $this->snoopy;
$snoopy->maxredirs = 0;
$snoopy->agent = '(compatible; MSIE 9; MSN 8; AOL 4.0; Windows 98)'; //模拟浏览器
if ($snoopy->fetch("http://www.$google_url/search?q=$search_worlds& start=0& num=10& hl=$language")) {
$response = $snoopy->results;
if (false !== stripos($response, '< h1 > 302 Moved< / h1 > ')) {
echo 302;
return false;
}
preg_match_all('/< h3 class = "r" > (.*)< \/h3>/smUi', $response, $store);
if (isset($store[1][0]) & & false !== stripos($store[1][0], $site_url)) {
$ismatch = stripos($store[1][0], $site_url);
$end = stripos($store[1][0], '& ', $ismatch);
$source = substr($store[1][0], $ismatch, $end);
$source = explode('& ', $source);
$source = $source[0];
if ($source == $site_url) {
$this->InfoMetas_model->update($embody->im_ic_id, 'meta_embody', '1');
$this->InfoSMS_model->add('info', $embody->im_ic_id, $embody->ic_author, 'lyy', '已收录');
echo 'ok';
} else {
$this->InfoMetas_model->update($embody->im_ic_id, 'meta_embody', date('Y-m-d H:i:s'));
$this->InfoSMS_model->add('info', $embody->im_ic_id, $embody->ic_author, 'lyy', '未被收录');
echo 'no';
}
} else {
$this->InfoMetas_model->update($embody->im_ic_id, 'meta_embody', date('Y-m-d H:i:s'));
$this->InfoSMS_model->add('info', $embody->im_ic_id, $embody->ic_author, 'lyy', '未被收录');
echo 'no';
}
}
echo '< br > end';
}
//定时批量更新CDN
public function update_cdn() {
$this->load->model('Logs_model');
@ -702,46 +632,6 @@ class Keyworlds extends CI_Controller {
var_dump($notice);
}
public function mobile_friendly_test() {
set_time_limit(60);
$url = $this->input->post('pageurl');
$googleapis = 'https://www.googleapis.com/pagespeedonline/v3beta1/mobileReady?url=' . $url;
$ch = curl_init();
curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22');
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_URL, $googleapis);
$response2 = curl_exec($ch);
$response = json_decode($response2);
$data['pageurl'] = $url;
$data['score'] = 60;
$data['pass'] = false;
isset($response->ruleGroups->USABILITY->score) & & $data['score'] = $response->ruleGroups->USABILITY->score;
isset($response->ruleGroups->USABILITY->pass) & & $data['pass'] = $response->ruleGroups->USABILITY->pass;
if (empty($response2)) {
$pass_text = '请求超时';
$title = '请求超时,请稍后再试!';
$link_color = 'text-error';
$data['score'] = '-';
} elseif ($data['pass']) {
$pass_text = 'OK';
$link_color = 'text-info';
$title = 'Awesome! This page is mobile-friendly (友好度' . $data['score'] . ', 总分100分)';
} else {
$pass_text = 'No';
$title = 'Not mobile-friendly (友好度' . $data['score'] . ', 总分100分)';
$link_color = 'text-error';
}
$html = '< a title = "' . $title . '" class = "' . $link_color . '" target = "__blank" href = "https://developers.google.com/speed/pagespeed/insights/?utm_source=analytics&tab=mobile&url=' . $data['pageurl'] . '" > < span > ' . $pass_text . ',友好度:' . $data['score'] . '< / span > < / a > ';
echo $html;
}
protected function curl($url, $useragent = 'PC', $referer = false) {
if ($useragent == 'PC') {
$useragent = "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22";
@ -755,13 +645,11 @@ class Keyworlds extends CI_Controller {
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
curl_setopt($ch, CURLOPT_FAILONERROR, false);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
// curl_setopt($ch, CURLOPT_SSLVERSION, 1); // 2,3
// curl_setopt($ch, CURLOPT_SSLVERSION, 'CURL_SSLVERSION_SSLv3');
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($ch, CURLOPT_USERAGENT, $useragent);
curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
curl_setopt($ch, CURLOPT_PROXY, "202.103.68.28:11111");
// curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
// curl_setopt($ch, CURLOPT_PROXY, "202.103.68.28:11111"); // 本地代理
if ($referer) {
curl_setopt($ch, CURLOPT_REFERER, $referer);
}
@ -772,11 +660,11 @@ class Keyworlds extends CI_Controller {
$reponse = curl_exec($ch);
if (curl_errno($ch)) {
log_message('error', "search engines curl error code: ".curl_error($ch).";");
log_message('error', "curl error code: ".curl_error($ch).";");
} else {
$httpStatusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (200 !== $httpStatusCode) {
log_message('error', "search engines Request html Status Code: ".$httpStatusCode.";");
log_message('error', "curl Request html Status Code: ".$httpStatusCode.";");
}
}
curl_close($ch);