diff --git a/server/app.js b/server/app.js index 6966598..4191137 100644 --- a/server/app.js +++ b/server/app.js @@ -11,7 +11,7 @@ const log4js = require('./config/log4'); const index = require('./routes/index') -const { Aids: syncAids, AidsState: syncAidsState, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs'); +const { Aids: syncAids, AidsState: syncAidsState, newHotelDetails: syncNewHotels, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs'); const rlog = require('./middleware/request_log'); // error handler @@ -43,8 +43,9 @@ app.use(async (ctx, next) => { }) // schedule jobs -syncAids(); +// syncAids(); syncAidsState(); +syncNewHotels(); syncHotelDetails(); syncChinas(); diff --git a/server/jobs/syncHeytripJobs.js b/server/jobs/syncHeytripJobs.js index 37a2001..cbba6c0 100644 --- a/server/jobs/syncHeytripJobs.js +++ b/server/jobs/syncHeytripJobs.js @@ -1,98 +1,151 @@ const { scheduleJob } = require('node-schedule'); const heytripService = require('../services/heytripService'); +/** + * node-schedule + * ? * * * * * * + ┬ ┬ ┬ ┬ ┬ ┬ + │ │ │ │ │ │ + │ │ │ │ │ └ day of week (0 - 7) (0 or 7 is Sun) + │ │ │ │ └───── month (1 - 12) + │ │ │ └────────── day of month (1 - 31) + │ │ └─────────────── hour (0 - 23) + │ └──────────────────── minute (0 - 59) + └───────────────────────── second (0 - 59, OPTIONAL) + */ + /** * 获取可用的酒店id */ const Aids = () => { return false; // const jobA = scheduleJob('*/2 * * * * *', async function () { - const jobA = scheduleJob('0 0 0 * * *', async function () { - console.log('syncing heytrip, get available accommodation ids.'); - const isRunning = jobA.pendingInvocations[0]?.job?.running == 1; - if (!isRunning) { - const res = await heytripService.syncAids(); - if (res.nextPage !== true) { - console.log('job completed! canceled job!'); - jobA.cancel(); - } - } else { - console.log('pre job running! cancelNext'); - jobA.cancelNext(); - } + // const jobA = scheduleJob('0 0 0 * * *', async function () { + // console.log('syncing heytrip, get available accommodation ids.'); + // const isRunning = jobA.pendingInvocations[0]?.job?.running == 1; + // if (!isRunning) { + // const res = await heytripService.syncAids(); + // if (res.nextPage !== true) { + // console.log('job completed! canceled job!'); + // jobA.cancel(); + // } + // } else { + // console.log('pre job running! cancelNext'); + // jobA.cancelNext(); + // } - }); + // }); }; /** * 更新酒店的状态, 是否下架 + * 每天启动同步; + * * 每次同步需要3000+页 */ const AidsState = () => { - const jobAS = scheduleJob('*/2 * * * * *', async function () { - // const jobAS = scheduleJob('0 5 0 * * *', async function () { - console.log('--------------------syncing heytrip, get available accommodation ids.--------------------'); - const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1; - if (!isRunning) { - const res = await heytripService.syncAidState(); - // jobAS.cancel(); // debug: 0 - if (res.nextPage !== true) { - console.log('job completed! canceled job[AidsState]!'); - jobAS.cancel(); + const dailyJob = scheduleJob('0 0 0 * * *', async function () { + const jobAS = scheduleJob('*/2 * * * * *', async function () { + console.log('--------------------syncing heytrip, get available accommodation ids.--------------------'); + const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1; + if (!isRunning) { + const res = await heytripService.syncAidState(); + // jobAS.cancel(); // debug: 0 + if (res.nextPage !== true) { + console.log('job completed! canceled job[AidsState]!'); + jobAS.cancel(); + } + } else { + console.log('pre job running! cancelNext[AidsState]'); + jobAS.cancelNext(); + } + }); + }); +}; + +/** + * 获取新增的酒店详情 + * ID同步结束后启动, ID大约需要3小时 + * 每天03:05:00启动 + */ +const newHotelDetails = () => { + // return false; + const dailyJob1 = scheduleJob('0 5 3 * * *', async function () { + const job1 = scheduleJob('*/5 * * * * *', async function () { + console.log('-------------------------syncing heytrip, get accommodation details.-------------------------'); + + const isRunning = job1.pendingInvocations[0]?.job?.running == 1; + if (!isRunning) { + const res = await heytripService.newHotels('1'); + + // job1.cancel(); // debug: 0 + if (res.next !== true) { + job1.cancel(res.restart); + console.log('job completed! canceled job[newHotelDetails]!'); + } + } else { + console.log('pre job running! cancelNext[newHotelDetails]'); + job1.cancelNext(); } - } else { - console.log('pre job running! cancelNext[AidsState]'); - jobAS.cancelNext(); - } + }); }); }; /** * 更新酒店详情, 按语种 + * 每周启动一次 + * 周五20:05:00启动 */ const hotelLgcDetails = () => { // return false; - const job2 = scheduleJob('*/4 * * * * *', async function () { - console.log('-------------------------syncing heytrip, get accommodation details.-------------------------'); + const dailyJob2 = scheduleJob('0 5 20 * * 5', async function () { + const job2 = scheduleJob('*/5 * * * * *', async function () { + console.log('-------------------------syncing heytrip, get accommodation details.-------------------------'); - const isRunning = job2.pendingInvocations[0]?.job?.running == 1; - if (!isRunning) { - const res = await heytripService.newHotelsLgc('1'); + const isRunning = job2.pendingInvocations[0]?.job?.running == 1; + if (!isRunning) { + const res = await heytripService.newHotelsLgc('1'); - // job2.cancel(); // debug: 0 - if (res.next !== true) { - job2.cancel(); - console.log('job completed! canceled job[hotelLgcDetails]!'); + // job2.cancel(); // debug: 0 + if (res.next !== true) { + job2.cancel(res.restart); + console.log('job completed! canceled job[hotelLgcDetails]!'); + } + } else { + console.log('pre job running! cancelNext[hotelLgcDetails]'); + job2.cancelNext(); } - } else { - console.log('pre job running! cancelNext[hotelLgcDetails]'); - job2.cancelNext(); - } + }); }); }; /** - * 更新中国酒店详情. 已完成 + * 更新中国酒店的中文详情. + * 每周启动一次 + * 周六20:05:00启动 */ const chinaHotelDetails = () => { - return false; - const job3 = scheduleJob('*/4 * * * * *', async function () { - console.log('syncing heytrip, get china accommodation details.'); - const isRunning = job3.pendingInvocations[0]?.job?.running == 1; - if (!isRunning) { - const res = await heytripService.chinaHotelsLgc2('2'); - if (res.next !== true) { - job3.cancel(); - console.log('job completed! canceled job!'); - // job3.reschedule('0 0 0 * * *'); + // return false; + const dailyJobCN = scheduleJob('0 5 20 * * 6', async function () { + const job3 = scheduleJob('*/4 * * * * *', async function () { + console.log('-------------------------syncing heytrip, get china accommodation details.-------------------------'); + const isRunning = job3.pendingInvocations[0]?.job?.running == 1; + if (!isRunning) { + const res = await heytripService.chinaHotelsLgc2('2'); + // job3.cancel(); // debug: 0 + if (res.next !== true) { + job3.cancel(res.restart); + console.log('job completed! canceled job[chinaHotelDetails]!'); + // job3.reschedule('0 0 0 * * *'); + } + } else { + console.log('pre job running! cancelNext'); + job3.cancelNext(); } - } else { - console.log('pre job running! cancelNext'); - job2.cancelNext(); - } + }); }); }; module.exports = { Aids, AidsState, - hotelLgcDetails, chinaHotelDetails + newHotelDetails, hotelLgcDetails, chinaHotelDetails } diff --git a/server/services/heytripService.js b/server/services/heytripService.js index 61338cd..7a73fa0 100644 --- a/server/services/heytripService.js +++ b/server/services/heytripService.js @@ -97,7 +97,7 @@ class Heytrip { // nest: true, }); - return { count, rows: rows.map((item) => item.dataValues) }; + return { count, rows: rows.map((item) => item.toJSON()) }; // return { count, rows }; }; @@ -110,6 +110,17 @@ class Heytrip { return ret; }; + +/** + * ************************************************************************************************************ + * 同步heytrip的酒店 + * 1. 获取可用的酒店id, 缺少的设为[99=下架] + * 2. 根据ID获取酒店详情, 无信息设为下架 + */ + /** + * @deprecated + * 第一次同步录库, 完成 + */ syncAids = async () => { const lastPageIndex = await this.getLastPageIndex(); const pageIndex = lastPageIndex + 1; @@ -131,6 +142,9 @@ class Heytrip { }; }; + /** + * 获取酒店ID + */ syncAidState = async () => { const today = new Date(); today.setHours(0, 0, 0, 0); // set the time to 00:00:00.000 @@ -140,7 +154,7 @@ class Heytrip { pageIndex = lastPageIndex; if (isEmpty(lastPageIndex)) { lastPageIndex = await this.getLastPageIndex({ last_modify_time: { [Op.gt]: today }, page_index: { [Op.lt]: 9999 } }); - console.log('syncAidState', 'lastPageIndex', lastPageIndex); + // console.log('syncAidState', 'lastPageIndex', lastPageIndex); pageIndex = lastPageIndex + 1; } @@ -187,11 +201,13 @@ class Heytrip { if (!isEmpty(stateNormal)) { await HeytripIds.update({ update_flag: 0, page_index: pageIndex, last_modify_time: Sequelize.fn('NOW') }, { where: { hotel_id: stateNormal } }); } + // 新增ID const newIds = validIds.filter((id) => !savedIds.map((item) => item.hotel_id).includes(id)); if (!isEmpty(newIds)) { - const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1 })); + const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1, priority: -10 })); await HeytripIds.bulkCreate(insertRows); } + // 页码滚动 const oldToNext = savedPageIds.filter((item) => !validIds.includes((item.hotel_id))).map((item) => item.hotel_id); if (!isEmpty(oldToNext)) { await HeytripIds.update({ page_index: Number(pageIndex)+9999, update_flag: 99 }, { where: { hotel_id: oldToNext } }); @@ -203,13 +219,29 @@ class Heytrip { }; } - newHotels = async () => { + /** + * 获取新添加的酒店 + * 在`syncAidState`中设置了`update_flag=1`, `priority=-10` + */ + newHotels = async (lgc) => { const [rows] = await Sequelize.query( - 'SELECT i.hotel_id, IFNULL(h.hi_sn ,0) info_exists FROM heytrip_ids as i LEFT JOIN hotelinfo AS h ON h.hotel_id = i.hotel_id WHERE h.hi_sn IS NULL AND update_flag != 0 ORDER BY info_exists LIMIT 10' + `SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists + FROM heytrip_ids AS i + LEFT JOIN hotelinfo2 AS h ON h.hotel_id = i.hotel_id + AND h.lgc = ${lgc} + WHERE h.hi2_sn IS NULL + AND update_flag = 1 + ORDER BY info_exists LIMIT 10` + , { logging: false } ); - return rows; + const res = await this.syncInitHotelLgcDetailsAction(rows, LGC_MAPPED[lgc]); + return res; }; + + /** + * 获取缺少的语种详情 + */ newHotelsLgc = async (lgc) => { const [rows] = await Sequelize.query( `SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists @@ -233,15 +265,21 @@ class Heytrip { const res = await this.syncInitHotelDetailsAction(rows, LGC_MAPPED['2']); return res; }; + + /** + * 中国酒店: 获取缺少的语种详情 + * * 中国: id > 20000000 + */ chinaHotelsLgc2 = async (lgc) => { const [rows] = await Sequelize.query( `SELECT i.hotel_id -- FROM hotelinfo AS h FROM heytrip_ids AS i LEFT JOIN hotelinfo2 AS h2 ON i.hotel_id =h2.hotel_id - AND h2.lgc = ${lgc} + AND h2.lgc = ${lgc} WHERE h2.hi2_sn IS NULL - AND i.hotel_id > 20000000 + AND i.hotel_id > 20000000 + AND i.update_flag != 99 LIMIT 10` , { logging: false } ); @@ -263,6 +301,7 @@ class Heytrip { /** * @deprecated + * 第一次录库, 已执行 */ syncInitHotelDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => { let allIds = []; @@ -309,11 +348,13 @@ class Heytrip { } }; + /** + * 录入酒店的信息 + * todo: 更新详情 + */ syncInitHotelLgcDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => { - let allIds = []; try { - allIds = rows.map((item) => item.hotel_id); - console.log('allIds', allIds); + const allIds = rows.map((item) => item.hotel_id); if (isEmpty(rows)) { return { next: !isEmpty(allIds), data: allIds }; @@ -321,7 +362,6 @@ class Heytrip { const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds }, }); const _BaseInfoExistsMapped = _BaseInfoExists.reduce((ru, c) => ({...ru, [`${c.hotel_id}`]: c }), {}); const existsIds = _BaseInfoExists.map((item) => `${item.hotel_id}`); - // console.log('existsIds', existsIds); const res = await AccommodationsDetails({ Language: lgcObj.locale, @@ -331,22 +371,30 @@ class Heytrip { // hotel info const insertData = resolveDetails(res, lgcObj); // return insertData; // debug: 0 - const resIds = insertData.info.map((item) => item.hotel_id); + const resIds = insertData.info.map((item) => `${item.hotel_id}`); + /** 开始Database */ const result = await Sequelize.transaction(async (transaction) => { const sequelizeOptions = { logging: false, transaction }; let Info; + /** + * 无返回数据, 设为失效`99` + * * 但部分中国酒店, 只有中文数据, 请求英文无返回 + */ const offInfo = allIds.filter((iitem) => !resIds.includes(`${iitem}`)); if (!isEmpty(offInfo)) { - await HeytripIds.update({ update_flag: 99 }, { where: { hotel_id: offInfo }, ...sequelizeOptions }); + const flag = Number(lgcObj.lgc) === 1 ? 2 : 99; // 等待获取中文数据 + const priority = Number(lgcObj.lgc) === 1 ? 0 : 99; + await HeytripIds.update({ update_flag: flag, priority }, { where: { hotel_id: offInfo }, ...sequelizeOptions }); } const updateInfo = insertData.info.filter((iitem) => existsIds.includes(`${iitem.hotel_id}`)); if (!isEmpty(updateInfo)) { - await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); + await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); for await (const updateRow of updateInfo) { + // 有中英文的, 把名称合并, eg.上海意家人酒店Yijiaren Hotel const _BaseName = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].hotel_name; if ((_BaseName || '').includes((updateRow.hotel_name || '').substring(0, 4))) { continue; @@ -354,9 +402,9 @@ class Heytrip { const _BaseAddress = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].address; await Hotelinfo.update( { - update_flag: 0, - hotel_name: lgcObj.lgc === 1 ? `${_BaseName}${updateRow.hotel_name}` : `${updateRow.hotel_name}${_BaseName}`, - address: lgcObj.lgc === 1 ? `${_BaseAddress}${updateRow.address}` : `${updateRow.address}${_BaseAddress}`, + update_flag: 0, priority: 0, + hotel_name: Number(lgcObj.lgc) === 1 ? `${_BaseName}${updateRow.hotel_name}` : `${updateRow.hotel_name}${_BaseName}`, + address: Number(lgcObj.lgc) === 1 ? `${_BaseAddress}${updateRow.address}` : `${updateRow.address}${_BaseAddress}`, }, { where: { hotel_id: updateRow.hotel_id }, ...sequelizeOptions } ); @@ -378,7 +426,8 @@ class Heytrip { if (!isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions); if (!isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions); - if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); + if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); + if (!isEmpty(updateInfo)) await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); return Info; }); @@ -387,10 +436,13 @@ class Heytrip { } catch (error) { console.log(error); - return { next: false, restart: true, data: allIds }; + return { next: false, restart: true, }; } }; + /** + * 获取实时的报价 + */ getHotelAvailability = async (param) => { const { hotel_id, checkin, checkout, adults, children_ages, rooms, nationality } = param; const paramBody = { @@ -427,4 +479,5 @@ class Heytrip { return await Logs.create(data, { logging: false }); }; } + module.exports = new Heytrip();