自动抓取，自动添加数据

application/third_party/pagespeed/controllers/index.php

@@ -16,12 +16,25 @@ class Index extends CI_Controller {
 
     //自动抓取分数和排名
     public function auto($sitecode = 'cht') {
-        $this->run('/', 'cht');
+        //添加数据，信息平台中发布的，并且当前数据库没有的
+        $this->PageSpeedData_model->insert_list($sitecode);
+        $update_list = $this->PageSpeedData_model->get_update_list($sitecode, 2);
+        if (empty($update_list)) {
+            echo 'all done!';
+            return;
+        }
+        foreach ($update_list as $item) {
+            $this->run($item->psd_URL, $item->psd_SiteCode);
+        }
     }
 
-    public function run($psd_URL, $psd_SiteCode) {
+    public function run($psd_URL = false, $psd_SiteCode = false) {
         if (empty($psd_URL) || empty($psd_SiteCode)) {
-            return false;
+            $psd_URL = $this->input->get_post('psd_URL');
+            $psd_SiteCode = $this->input->get_post('psd_SiteCode');
+            if (empty($psd_URL) || empty($psd_SiteCode)) {
+                return false;
+            }
         }
         $sites = $this->config->item('site');
         if (empty($sites[$psd_SiteCode])) {
@@ -33,9 +46,9 @@ class Index extends CI_Controller {
         $PageSpeedData = new StdClass;
 
         $desktop_url = "https://www.googleapis.com/pagespeedonline/v2/runPagespeed?strategy=desktop&locale=zh_CN&url=$run_URL";
-        $desktop_url = "http://pagespeed.mycht.cn/pagespeedonline/v2/runPagespeed?strategy=desktop&locale=zh_CN&url=$run_URL";
+        // $desktop_url = "http://pagespeed.mycht.cn/pagespeedonline/v2/runPagespeed?strategy=desktop&locale=zh_CN&url=$run_URL";
         $mobile_url = "https://www.googleapis.com/pagespeedonline/v2/runPagespeed?strategy=mobile&locale=zh_CN&url=$run_URL";
-        $mobile_url = "http://pagespeed.mycht.cn/pagespeedonline/v2/runPagespeed?strategy=mobile&locale=zh_CN&url=$run_URL";
+        //$mobile_url = "http://pagespeed.mycht.cn/pagespeedonline/v2/runPagespeed?strategy=mobile&locale=zh_CN&url=$run_URL";
         //echo $desktop_url;die();
         $desktop_data = GET_HTTP($desktop_url);
         $mobile_data = GET_HTTP($mobile_url);
@@ -45,14 +58,19 @@ class Index extends CI_Controller {
         $desktop_object = json_decode($desktop_data);
         $mobile_object = json_decode($mobile_data);
         if (!empty($desktop_object->error) || !empty($mobile_object->error)) {
+            $this->PageSpeedData_model->update_error($pagespeed_data->psd_id);
+            echo 'pagespeed_error:' . json_encode($desktop_object) . ' ' . json_encode($mobile_object);
+            log_message('error ', 'pagespeed_error:' . json_encode($desktop_object) . ' ' . json_encode($mobile_object));
             return false;
         }
 
         if (!$desktop_object->responseCode == '200' || !$mobile_object->responseCode == '200') {
-            echo 'ERROR:' . $desktop_object->responseCode . ' ' . $mobile_object->responseCode;
+            $this->PageSpeedData_model->update_error($pagespeed_data->psd_id);
+            echo 'pagespeed_error:' . json_encode($desktop_object) . ' ' . json_encode($mobile_object);
+            log_message('error ', 'pagespeed_error:' . json_encode($desktop_object) . ' ' . json_encode($mobile_object));
             return FALSE;
         }
-        
+
         $PageSpeedData->psd_DesktopScore = $desktop_object->ruleGroups->SPEED->score;
         $PageSpeedData->psd_MobileScore = $mobile_object->ruleGroups->SPEED->score;
         $PageSpeedData->psd_DesktopData = $desktop_data;
@@ -61,14 +79,14 @@ class Index extends CI_Controller {
         $PageSpeedData->psd_Datetime = date('Y-m-d H:i:s', time());
         $PageSpeedData->psd_SiteCode = $psd_SiteCode;
         $PageSpeedData->psd_ErrorCount = 0;
-        
+
         if (empty($pagespeed_data)) {
             $psd_id = $this->PageSpeedData_model->add('PageSpeedData', $PageSpeedData);
         } else {
             $where = array('psd_id' => $pagespeed_data->psd_id);
             $this->PageSpeedData_model->update('PageSpeedData', $PageSpeedData, $where);
         }
-        echo $run_URL . '<br/>';
+        echo json_encode($pagespeed_data);
     }
 
 }

application/third_party/pagespeed/models/PageSpeedData_model.php

@@ -22,7 +22,7 @@ class PageSpeedData_model extends CI_Model {
         $this->init();
         $this->topnum = 1;
         $this->where = ' AND  psd.psd_URL=' . $this->INFO->escape($psd_URL);
-        $this->where = ' AND  psd.psd_SiteCode=' . $this->INFO->escape($psd_SiteCode);
+        $this->where .= ' AND  psd.psd_SiteCode=' . $this->INFO->escape($psd_SiteCode);
         return $this->get_list();
     }
 
@@ -34,10 +34,64 @@ class PageSpeedData_model extends CI_Model {
         }
     }
 
+    public function insert_list($psd_SiteCode) {
+        $sql = "
+                INSERT INTO PageSpeedData
+                  (
+                    psd_URL
+                   ,psd_SiteCode
+                  )
+                SELECT TOP 10 ic.ic_url
+                      ,ic.ic_sitecode
+                FROM   Tourmanager.dbo.infoContents ic
+                WHERE  ic.ic_status = 1
+                       AND ic.ic_url<>''
+                       AND ic.ic_sitecode = ?
+                       AND NOT EXISTS(
+                               SELECT TOP 1 1
+                               FROM   PageSpeedData psd
+                               WHERE  psd.psd_URL = ic.ic_url
+                                      AND psd.psd_SiteCode = ic.ic_sitecode
+                           )
+                ORDER BY
+                       ic.ic_datetime DESC
+            ";
+        $this->INFO->query($sql,array($psd_SiteCode));
+    }
+
+    public function get_update_list($psd_SiteCode, $topnum = 3) {
+        $this->init();
+        $this->topnum = $topnum;
+        $this->where = ' AND  psd.psd_SiteCode=' . $this->INFO->escape($psd_SiteCode);
+        $this->where .= ' AND  ISNULL(psd.psd_Datetime,1999) < GETDATE()-1 ';
+        $this->orderby = ' ORDER BY psd_Datetime ASC ';
+        return $this->get_list();
+    }
+
     public function update($table, $data, $where) {
         $this->INFO->update($table, $data, $where);
     }
 
+    //记录错误次数
+    public function update_error($psd_id) {
+        $sql = "   
+            UPDATE PageSpeedData
+            SET    psd_ErrorCount     = psd_ErrorCount+1
+                    , psd_Datetime=GETDATE()
+            WHERE  psd_id             = ?
+            ";
+        $this->INFO->query($sql, array($psd_id));
+        //删除错误数大于3的数据，保持没有冗余数据
+        $sql = "  DELETE FROM PageSpeedData WHERE psd_ErrorCount>3 ";
+        $this->INFO->query($sql);
+    }
+
+    //删除数据
+    //  public function delete($psd_id) {
+    //     $sql = "  DELETE FROM PageSpeedData WHERE psd_id=? ";
+    //      return $this->INFO->query($sql, array($psd_id));
+    //  }
+
     public function get_list() {
         $this->topnum ? $sql = "SELECT TOP " . $this->topnum : $sql = "SELECT ";
         $sql .= "