From a0cdcc28c1785c0c62c38562d393b86759b89493 Mon Sep 17 00:00:00 2001 From: Lei OT Date: Wed, 21 Aug 2024 14:40:11 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=90=8C=E6=AD=A5:=20=E5=90=8C?= =?UTF-8?q?=E6=AD=A5ID,=20=E6=9B=B4=E6=96=B0=E4=B8=8B=E6=9E=B6=E7=8A=B6?= =?UTF-8?q?=E6=80=81;=20=E6=96=B0=E5=A2=9E=E7=9A=84=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/app.js | 9 +- server/jobs/syncHeytripJobs.js | 57 +++++++-- server/services/heytripService.js | 197 ++++++++++++++++++++++-------- server/vendor/heytrip.js | 2 +- 4 files changed, 200 insertions(+), 65 deletions(-) diff --git a/server/app.js b/server/app.js index 8c6433a..a333b44 100644 --- a/server/app.js +++ b/server/app.js @@ -9,7 +9,7 @@ const logger = require('koa-logger') const index = require('./routes/index') -const { Aids: syncAids, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs'); +const { Aids: syncAids, AidsState: syncAidsState, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs'); const rlog = require('./middleware/request_log'); // error handler @@ -41,9 +41,10 @@ app.use(async (ctx, next) => { }) // schedule jobs -// syncAids(); -// syncHotelDetails(); -// syncChinas(); +syncAids(); +syncAidsState(); +syncHotelDetails(); +syncChinas(); // routes app.use(index.routes(), index.allowedMethods()) diff --git a/server/jobs/syncHeytripJobs.js b/server/jobs/syncHeytripJobs.js index a5aef67..73f8703 100644 --- a/server/jobs/syncHeytripJobs.js +++ b/server/jobs/syncHeytripJobs.js @@ -1,29 +1,63 @@ const { scheduleJob } = require('node-schedule'); const heytripService = require('../services/heytripService'); -const { LGC_MAPPED } = require('../config/constants'); +/** + * 获取可用的酒店id + */ const Aids = () => { - const job = scheduleJob('*/2 * * * * *', async function () { + return false; + // const jobA = scheduleJob('*/2 * * * * *', async function () { + const jobA = scheduleJob('0 0 0 * * *', async function () { console.log('syncing heytrip, get available accommodation ids.'); + const isRunning = jobA.pendingInvocations[0]?.job?.running == 1; + if (!isRunning) { + const res = await heytripService.syncAids(); + if (res.nextPage !== true) { + console.log('job completed! canceled job!'); + jobA.cancel(); + } + } else { + console.log('pre job running! cancelNext'); + jobA.cancelNext(); + } + + }); +}; - const res = await heytripService.syncAids(); - if (res.nextPage !== true) { - job.cancel(); +/** + * 更新酒店的状态, 是否下架 + */ +const AidsState = () => { + // const jobAS = scheduleJob('*/2 * * * * *', async function () { + const jobAS = scheduleJob('0 5 0 * * *', async function () { + console.log('--------------------syncing heytrip, get available accommodation ids.--------------------'); + const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1; + if (!isRunning) { + const res = await heytripService.syncAidState(); + if (res.nextPage !== true) { + console.log('job completed! canceled job!'); + jobAS.cancel(); + } + } else { + console.log('pre job running! cancelNext'); + jobAS.cancelNext(); } }); }; +/** + * 更新酒店详情, 按语种 + */ const hotelLgcDetails = () => { + // return false; const job2 = scheduleJob('*/4 * * * * *', async function () { console.log('-------------------------syncing heytrip, get accommodation details.-------------------------'); const isRunning = job2.pendingInvocations[0]?.job?.running == 1; if (!isRunning) { - // const res = await heytripService.syncHotelDetails(); - // const res = await heytripService.syncInitHotelLgcDetailsAction(LGC_MAPPED['1']); const res = await heytripService.newHotelsLgc('1'); - job2.cancel(); // debug: 0 + // job2.cancel(); // debug: 0 if (res.next !== true) { job2.cancel(); console.log('job completed! canceled job!'); @@ -35,7 +69,11 @@ const hotelLgcDetails = () => { }); }; +/** + * 更新中国酒店详情. 已完成 + */ const chinaHotelDetails = () => { + return false; const job3 = scheduleJob('*/4 * * * * *', async function () { console.log('syncing heytrip, get china accommodation details.'); const isRunning = job3.pendingInvocations[0]?.job?.running == 1; @@ -54,5 +92,6 @@ const chinaHotelDetails = () => { }; module.exports = { - Aids, hotelLgcDetails, chinaHotelDetails + Aids, AidsState, + hotelLgcDetails, chinaHotelDetails } diff --git a/server/services/heytripService.js b/server/services/heytripService.js index d707e25..12a839c 100644 --- a/server/services/heytripService.js +++ b/server/services/heytripService.js @@ -3,7 +3,7 @@ const initModels = require('./../models/init-models'); const { AvailableAccommodationIds, AccommodationsDetails, Availability } = require('../vendor/heytrip'); const { isEmpty } = require('../utils/commons'); const { resolveDetails, resolveRatePlans } = require('../helper/heytripDataHelper'); -const { DEFAULT_LGC, LGC_MAPPED } = require('../config/constants'); +const { DEFAULT_LGC, LGC_MAPPED, HEYTRIP_API_PROD } = require('../config/constants'); const { sequelize: Sequelize, Op } = db; @@ -24,6 +24,9 @@ const ReviewsSummaries = models.reviewsSummariesModel; const Locations = models.locationsModel; // const CacheAvailability = models.cacheAvailabilityModel; +const Logs = models.requestLogsModel; + + // HeytripIds.hasMany(Hotelinfo, { // foreignKey: 'hotel_id', // onDelete: 'NO ACTION', @@ -32,7 +35,8 @@ const Locations = models.locationsModel; // Hotelinfo.belongsTo(HeytripIds, { as: 'aid', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' }); // Hotelinfo.hasMany(Rooms, { sourceKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' }); // Rooms.belongsTo(Hotelinfo, { targetKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', }); -Hotelinfo.hasMany(Hotelinfo2, { as: 'lgc_info', sourceKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' }); +Hotelinfo.belongsTo(HeytripIds, { as: 'aid', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' }); +Hotelinfo.hasMany(Hotelinfo2, { as: 'locale_info', sourceKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' }); Hotelinfo.hasMany(City, { as: 'city', sourceKey: 'city_id', foreignKey: 'id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', }); // 多语种, 所以实际是 hasMany , 用 hasOne 要指定 lgc= 1 或者2 Hotelinfo.hasMany(Country, { as: 'country', sourceKey: 'country_code', foreignKey: 'id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', }); // 多语种, 所以实际是 hasMany , 用 hasOne 要指定 lgc= 1 或者2 // Hotelinfo2.belongsTo(Hotelinfo, { targetKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', }); @@ -45,8 +49,8 @@ class Heytrip { const keywordSplit = keyword.split(' '); const keywordWhere = (field) => keywordSplit.map((word) => Sequelize.where(Sequelize.fn('instr', Sequelize.col(field), word), { [Op.gt]: 0 })); // 'hotelinfo.address' - const keywordSearch = ['hotelinfo.hotel_name', ].map((field) => ({ [Op.and]: keywordWhere(field) })); - const keywordOrder = ['hotelinfo.hotel_name',].reduce((ro, field) => ro.concat(keywordSplit.map((word) => Sequelize.fn('instr', Sequelize.col(field), word))), []); + const keywordSearch = ['hotelinfo.hotel_name'].map((field) => ({ [Op.and]: keywordWhere(field) })); + const keywordOrder = ['hotelinfo.hotel_name'].reduce((ro, field) => ro.concat(keywordSplit.map((word) => Sequelize.fn('instr', Sequelize.col(field), word))), []); // const keywordSearchCount = ['hi2_hotel_name', 'hi2_address'].map((field) => ({ [Op.and]: keywordWhere(field) })); // const countRows = await Hotelinfo2.count({ @@ -62,9 +66,10 @@ class Heytrip { const { count, rows } = await Hotelinfo.findAndCountAll({ // const [ rows ] = await Hotelinfo.findAll({ include: [ + { model: HeytripIds, as: 'aid', attributes: ['hotel_id', 'update_flag'], where: { update_flag: { [Op.ne]: 99 } } }, { model: Hotelinfo2, - as: 'lgc_info', + as: 'locale_info', attributes: [ ['hi2_sn', 'sn'], 'hotel_id', @@ -79,8 +84,8 @@ class Heytrip { required: false, separate: true, }, - { model: City, as: 'city', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true, }, - { model: Country, as: 'country', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true, }, + { model: City, as: 'city', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true }, + { model: Country, as: 'country', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true }, ], // include: [{ model: Hotelinfo2, as: 'h2', required: true }], where: { @@ -97,8 +102,12 @@ class Heytrip { // return { count, rows }; }; - getLastPageIndex = async () => { - const ret = await HeytripIds.max('page_index'); + getLastPageIndex = async (where = {}) => { + const ret = await HeytripIds.max('page_index', { where }); + return ret; + }; + getFirstPageIndex = async (where = {}) => { + const ret = await HeytripIds.min('page_index', { where }); return ret; }; @@ -123,6 +132,76 @@ class Heytrip { }; }; + syncAidState = async () => { + const today = new Date(); + today.setHours(0, 0, 0, 0); // set the time to 00:00:00.000 + + let lastPageIndex; let pageIndex; + lastPageIndex = await this.getFirstPageIndex({ last_modify_time: { [Op.lt]: today } }); + pageIndex = lastPageIndex; + if (isEmpty(lastPageIndex)) { + lastPageIndex = await this.getLastPageIndex({ last_modify_time: { [Op.gt]: today }, page_index: { [Op.lt]: 9999 } }); + pageIndex = lastPageIndex + 1; + } + console.log('syncAidState', lastPageIndex, pageIndex); + + const validIds = (await AvailableAccommodationIds(pageIndex)).map((id) => String(id)); + + if (isEmpty(validIds)) { + // await HeytripIds.update({ update_flag: 99, priority: 99 }, { where: { pageIndex: [Op.gt]: pageIndex } }); + // 同步结束; 本次没有的ID: 更新: 99=失效 + const stateOff = await HeytripIds.findAll({ + raw: true, + where: { + update_flag: { [Op.notIn]: [0, 1] }, + last_modify_time: { [Op.gt]: today }, + }, + attributes: ['hotel_id'], + }); + const stateOffIds = stateOff.map((item) => item.hotel_id); + if (!isEmpty(stateOffIds)) { + await HeytripIds.update({ update_flag: 99, priority: 99 }, { where: { hotel_id: stateOffIds } }); + console.log('updated stateOff', stateOffIds.length); + } + return { + nextPage: false, + }; + } + + const savedIds = await HeytripIds.findAll({ + raw: true, + where: { hotel_id: validIds }, + attributes: ['hotel_id'], + logging: false, + }); // savedIds <= validIds + const savedPageIds = await HeytripIds.findAll({ + raw: true, + where: { page_index: pageIndex, }, + attributes: ['hotel_id'], + // logging: false, + }); + + // 已存在ID: 更新: 状态, 页码, 时间 + const stateNormal = savedIds.filter((item) => validIds.includes((item.hotel_id))).map((item) => item.hotel_id); + if (!isEmpty(stateNormal)) { + await HeytripIds.update({ update_flag: 0, page_index: pageIndex, last_modify_time: Sequelize.fn('NOW') }, { where: { hotel_id: stateNormal } }); + } + const newIds = validIds.filter((id) => !savedIds.map((item) => item.hotel_id).includes(id)); + if (!isEmpty(newIds)) { + const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1 })); + await HeytripIds.bulkCreate(insertRows); + } + const oldToNext = savedPageIds.filter((item) => !validIds.includes((item.hotel_id))).map((item) => item.hotel_id); + if (!isEmpty(oldToNext)) { + await HeytripIds.update({ page_index: Number(pageIndex)+9999, update_flag: 99 }, { where: { hotel_id: oldToNext } }); + } + + return { + nextPage: true, + pageIndex, + }; + } + newHotels = async () => { const [rows] = await Sequelize.query( 'SELECT i.hotel_id, IFNULL(h.hi_sn ,0) info_exists FROM heytrip_ids as i LEFT JOIN hotelinfo AS h ON h.hotel_id = i.hotel_id WHERE h.hi_sn IS NULL AND update_flag != 0 ORDER BY info_exists LIMIT 10' @@ -131,13 +210,6 @@ class Heytrip { }; newHotelsLgc = async (lgc) => { - // const [rows] = await Sequelize.query( - // `SELECT h.hotel_id - // FROM hotelinfo AS h - // LEFT JOIN hotelinfo2 AS h2 ON h.hotel_id =h2.hotel_id - // AND h2.lgc = ${lgc} - // WHERE h2.hi2_sn IS NULL LIMIT 10` - // ); const [rows] = await Sequelize.query( `SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists FROM heytrip_ids AS i @@ -146,6 +218,7 @@ class Heytrip { WHERE h.hi2_sn IS NULL AND update_flag != 99 ORDER BY info_exists LIMIT 10` + , { logging: false } ); const res = await this.syncInitHotelLgcDetailsAction(rows, LGC_MAPPED[lgc]); return res; @@ -182,10 +255,12 @@ class Heytrip { return res; }; + /** + * @deprecated + */ syncInitHotelDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => { let allIds = []; try { - allIds = rows.map((item) => item.hotel_id); const updateIds = rows.filter((item) => item.info_exists !== 0).map((item) => item.hotel_id); const newIds = rows.filter((item) => item.info_exists === 0).map((item) => item.hotel_id); @@ -204,43 +279,40 @@ class Heytrip { // return insertData; // debug: 0 /** 开始Database */ - const sequelizeOptions = { logging: false }; - const result = await Sequelize.transaction(async t => { + const result = await Sequelize.transaction(async (transaction) => { + const sequelizeOptions = { logging: false, transaction }; let Info; - if ( ! isEmpty(insertData.info)) Info = await Hotelinfo.bulkCreate(insertData.info, sequelizeOptions); - if ( ! isEmpty(insertData.info2)) await Hotelinfo2.bulkCreate(insertData.info2, sequelizeOptions); - if ( ! isEmpty(insertData.rooms)) await Rooms.bulkCreate(insertData.rooms, sequelizeOptions); - if ( ! isEmpty(insertData.images)) await Images.bulkCreate(insertData.images, sequelizeOptions); - if ( ! isEmpty(insertData.facility)) await Facility.bulkCreate(insertData.facility, sequelizeOptions); - if ( ! isEmpty(insertData.infos)) await Informations.bulkCreate(insertData.infos, sequelizeOptions); - if ( ! isEmpty(insertData.reviews)) await Reviews.bulkCreate(insertData.reviews, sequelizeOptions); - if ( ! isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions); - if ( ! isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions); - if ( ! isEmpty(allIds)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: allIds } }); + if (!isEmpty(insertData.info)) Info = await Hotelinfo.bulkCreate(insertData.info, sequelizeOptions); + if (!isEmpty(insertData.info2)) await Hotelinfo2.bulkCreate(insertData.info2, sequelizeOptions); + if (!isEmpty(insertData.rooms)) await Rooms.bulkCreate(insertData.rooms, sequelizeOptions); + if (!isEmpty(insertData.images)) await Images.bulkCreate(insertData.images, sequelizeOptions); + if (!isEmpty(insertData.facility)) await Facility.bulkCreate(insertData.facility, sequelizeOptions); + if (!isEmpty(insertData.infos)) await Informations.bulkCreate(insertData.infos, sequelizeOptions); + if (!isEmpty(insertData.reviews)) await Reviews.bulkCreate(insertData.reviews, sequelizeOptions); + if (!isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions); + if (!isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions); + if (!isEmpty(allIds)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: allIds } }); return Info; }); return { next: !isEmpty(allIds), data: allIds }; - } catch (error) { console.log(error); return { next: false, data: allIds }; } - } + }; syncInitHotelLgcDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => { let allIds = []; try { - allIds = rows.map((item) => item.hotel_id); if (isEmpty(rows)) { return { next: !isEmpty(allIds), data: allIds }; } - const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds } }); + const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds }, logging: false }); const updateIds = _BaseInfoExists.map((item) => `${item.hotel_id}`); - console.log('updateIds', updateIds); - + // console.log('updateIds', updateIds); const res = await AccommodationsDetails({ Language: lgcObj.locale, @@ -253,30 +325,37 @@ class Heytrip { // return insertData; // debug: 0 /** 开始Database */ - const result = await Sequelize.transaction(async transaction => { + const result = await Sequelize.transaction(async (transaction) => { const sequelizeOptions = { logging: false, transaction }; let Info; - const newInfo = insertData.info.filter(iitem => !updateIds.includes(`${iitem.hotel_id}`)); - console.log('newInfo', newInfo.map(xx => xx.hotel_id)); - if ( ! isEmpty(newInfo)) Info = await Hotelinfo.bulkCreate(newInfo, sequelizeOptions); - - if ( ! isEmpty(insertData.info2)) await Hotelinfo2.bulkCreate(insertData.info2, sequelizeOptions); - if ( ! isEmpty(insertData.rooms)) await Rooms.bulkCreate(insertData.rooms, sequelizeOptions); - if ( ! isEmpty(insertData.images)) await Images.bulkCreate(insertData.images, sequelizeOptions); - if ( ! isEmpty(insertData.facility)) await Facility.bulkCreate(insertData.facility, sequelizeOptions); - if ( ! isEmpty(insertData.infos)) await Informations.bulkCreate(insertData.infos, sequelizeOptions); - if ( ! isEmpty(insertData.reviews)) await Reviews.bulkCreate(insertData.reviews, sequelizeOptions); - if ( ! isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions); - if ( ! isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions); - - if ( ! isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: newInfo.map(x => x.hotel_id) } }); + const offInfo = allIds.filter((iitem) => !resIds.includes(`${iitem}`)); + if (!isEmpty(offInfo)) { + await HeytripIds.update({ update_flag: 99 }, { where: { hotel_id: offInfo }, ...sequelizeOptions }); + } + + const newInfo = insertData.info.filter((iitem) => !updateIds.includes(`${iitem.hotel_id}`)); + // console.log( + // 'newInfo', + // newInfo.map((xx) => xx.hotel_id) + // ); + if (!isEmpty(newInfo)) Info = await Hotelinfo.bulkCreate(newInfo, sequelizeOptions); + + if (!isEmpty(insertData.info2)) await Hotelinfo2.bulkCreate(insertData.info2, sequelizeOptions); + if (!isEmpty(insertData.rooms)) await Rooms.bulkCreate(insertData.rooms, sequelizeOptions); + if (!isEmpty(insertData.images)) await Images.bulkCreate(insertData.images, sequelizeOptions); + if (!isEmpty(insertData.facility)) await Facility.bulkCreate(insertData.facility, sequelizeOptions); + if (!isEmpty(insertData.infos)) await Informations.bulkCreate(insertData.infos, sequelizeOptions); + if (!isEmpty(insertData.reviews)) await Reviews.bulkCreate(insertData.reviews, sequelizeOptions); + if (!isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions); + if (!isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions); + + if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); return Info; }); return { next: !isEmpty(allIds), data: allIds }; - } catch (error) { console.log(error); @@ -303,5 +382,21 @@ class Heytrip { const quoteRes = resolveRatePlans(_quoteRes); return quoteRes; }; + + writeHeytripRequestLog = async ({ action, body }) => { + const actionMapped = { + 'getHotelAvailability': '/Accommodation/Availability', + 'getHotelInfo': '/Accommodation/AccommodationsDetails', + 'getHotelIds': '/Accommodation/AvailableAccommodationIds', + }; + const data = { + action: action, + // method: ctx.method, + path: actionMapped[action], + request_data: body, + // ip: ctx.ip, + }; + return await Logs.create(data, { logging: false }); + }; } module.exports = new Heytrip(); diff --git a/server/vendor/heytrip.js b/server/vendor/heytrip.js index b91f58d..329c2da 100644 --- a/server/vendor/heytrip.js +++ b/server/vendor/heytrip.js @@ -23,7 +23,7 @@ const AvailableAccommodationIds = async (pageIndex) => { }, }); console.log('Call pageIndex', pageIndex, response.data.TotalPage); - return response.data.Data; + return response.data.Data || []; }; const AccommodationsDetails = async (body) => {