Haina
/
heytripgo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

conf: 同步触发规则

Browse Source
main
Lei OT 10 months ago
parent 997e9adbeb
commit 773ef97a27
3 changed files with 186 additions and 79 deletions
Show Stats Download Patch File Download Diff File

5
server/app.js
Unescape Escape View File

@ -11,7 +11,7 @@ const log4js = require('./config/log4');
const index = require('./routes/index') const index = require('./routes/index')
const { Aids: syncAids, AidsState: syncAidsState, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs'); const { Aids: syncAids, AidsState: syncAidsState, newHotelDetails: syncNewHotels, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs');
const rlog = require('./middleware/request_log'); const rlog = require('./middleware/request_log');
// error handler // error handler
@ -43,8 +43,9 @@ app.use(async (ctx, next) => {
}) })
// schedule jobs // schedule jobs
syncAids(); // syncAids();
syncAidsState(); syncAidsState();
syncNewHotels();
syncHotelDetails(); syncHotelDetails();
syncChinas(); syncChinas();

101
server/jobs/syncHeytripJobs.js
Unescape Escape View File

@ -1,35 +1,50 @@
const { scheduleJob } = require('node-schedule'); const { scheduleJob } = require('node-schedule');
const heytripService = require('../services/heytripService'); const heytripService = require('../services/heytripService');
/**
* node-schedule
* ? * * * * * *
┬ ┬ ┬ ┬ ┬ ┬
│ │ │ │ │ │
│ │ │ │ │ └ day of week (0 - 7) (0 or 7 is Sun)
│ │ │ │ └───── month (1 - 12)
│ │ │ └────────── day of month (1 - 31)
│ │ └─────────────── hour (0 - 23)
│ └──────────────────── minute (0 - 59)
└───────────────────────── second (0 - 59, OPTIONAL)
*/
/** /**
* 获取可用的酒店id * 获取可用的酒店id
*/ */
const Aids = () => { const Aids = () => {
return false; return false;
// const jobA = scheduleJob('*/2 * * * * *', async function () { // const jobA = scheduleJob('*/2 * * * * *', async function () {
const jobA = scheduleJob('0 0 0 * * *', async function () { // const jobA = scheduleJob('0 0 0 * * *', async function () {
console.log('syncing heytrip, get available accommodation ids.'); // console.log('syncing heytrip, get available accommodation ids.');
const isRunning = jobA.pendingInvocations[0]?.job?.running == 1; // const isRunning = jobA.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) { // if (!isRunning) {
const res = await heytripService.syncAids(); // const res = await heytripService.syncAids();
if (res.nextPage !== true) { // if (res.nextPage !== true) {
console.log('job completed! canceled job!'); // console.log('job completed! canceled job!');
jobA.cancel(); // jobA.cancel();
} // }
} else { // } else {
console.log('pre job running! cancelNext'); // console.log('pre job running! cancelNext');
jobA.cancelNext(); // jobA.cancelNext();
} // }
}); // });
}; };
/** /**
* 更新酒店的状态, 是否下架 * 更新酒店的状态, 是否下架
* 每天启动同步;
* * 每次同步需要3000+页
*/ */
const AidsState = () => { const AidsState = () => {
const dailyJob = scheduleJob('0 0 0 * * *', async function () {
const jobAS = scheduleJob('*/2 * * * * *', async function () { const jobAS = scheduleJob('*/2 * * * * *', async function () {
// const jobAS = scheduleJob('0 5 0 * * *', async function () {
console.log('--------------------syncing heytrip, get available accommodation ids.--------------------'); console.log('--------------------syncing heytrip, get available accommodation ids.--------------------');
const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1; const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) { if (!isRunning) {
@ -44,14 +59,46 @@ const AidsState = () => {
jobAS.cancelNext(); jobAS.cancelNext();
} }
}); });
});
};
/**
* 获取新增的酒店详情
* ID同步结束后启动, ID大约需要3小时
* 每天03:05:00启动
*/
const newHotelDetails = () => {
// return false;
const dailyJob1 = scheduleJob('0 5 3 * * *', async function () {
const job1 = scheduleJob('*/5 * * * * *', async function () {
console.log('-------------------------syncing heytrip, get accommodation details.-------------------------');
const isRunning = job1.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.newHotels('1');
// job1.cancel(); // debug: 0
if (res.next !== true) {
job1.cancel(res.restart);
console.log('job completed! canceled job[newHotelDetails]!');
}
} else {
console.log('pre job running! cancelNext[newHotelDetails]');
job1.cancelNext();
}
});
});
}; };
/** /**
* 更新酒店详情, 按语种 * 更新酒店详情, 按语种
* 每周启动一次
* 周五20:05:00启动
*/ */
const hotelLgcDetails = () => { const hotelLgcDetails = () => {
// return false; // return false;
const job2 = scheduleJob('*/4 * * * * *', async function () { const dailyJob2 = scheduleJob('0 5 20 * * 5', async function () {
const job2 = scheduleJob('*/5 * * * * *', async function () {
console.log('-------------------------syncing heytrip, get accommodation details.-------------------------'); console.log('-------------------------syncing heytrip, get accommodation details.-------------------------');
const isRunning = job2.pendingInvocations[0]?.job?.running == 1; const isRunning = job2.pendingInvocations[0]?.job?.running == 1;
@ -60,7 +107,7 @@ const hotelLgcDetails = () => {
// job2.cancel(); // debug: 0 // job2.cancel(); // debug: 0
if (res.next !== true) { if (res.next !== true) {
job2.cancel(); job2.cancel(res.restart);
console.log('job completed! canceled job[hotelLgcDetails]!'); console.log('job completed! canceled job[hotelLgcDetails]!');
} }
} else { } else {
@ -68,31 +115,37 @@ const hotelLgcDetails = () => {
job2.cancelNext(); job2.cancelNext();
} }
}); });
});
}; };
/** /**
* 更新中国酒店详情. 已完成 * 更新中国酒店的中文详情.
* 每周启动一次
* 周六20:05:00启动
*/ */
const chinaHotelDetails = () => { const chinaHotelDetails = () => {
return false; // return false;
const dailyJobCN = scheduleJob('0 5 20 * * 6', async function () {
const job3 = scheduleJob('*/4 * * * * *', async function () { const job3 = scheduleJob('*/4 * * * * *', async function () {
console.log('syncing heytrip, get china accommodation details.'); console.log('-------------------------syncing heytrip, get china accommodation details.-------------------------');
const isRunning = job3.pendingInvocations[0]?.job?.running == 1; const isRunning = job3.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) { if (!isRunning) {
const res = await heytripService.chinaHotelsLgc2('2'); const res = await heytripService.chinaHotelsLgc2('2');
// job3.cancel(); // debug: 0
if (res.next !== true) { if (res.next !== true) {
job3.cancel(); job3.cancel(res.restart);
console.log('job completed! canceled job!'); console.log('job completed! canceled job[chinaHotelDetails]!');
// job3.reschedule('0 0 0 * * *'); // job3.reschedule('0 0 0 * * *');
} }
} else { } else {
console.log('pre job running! cancelNext'); console.log('pre job running! cancelNext');
job2.cancelNext(); job3.cancelNext();
} }
}); });
});
}; };
module.exports = { module.exports = {
Aids, AidsState, Aids, AidsState,
hotelLgcDetails, chinaHotelDetails newHotelDetails, hotelLgcDetails, chinaHotelDetails
} }

89
server/services/heytripService.js
Unescape Escape View File

@ -97,7 +97,7 @@ class Heytrip {
// nest: true, // nest: true,
}); });
return { count, rows: rows.map((item) => item.dataValues) }; return { count, rows: rows.map((item) => item.toJSON()) };
// return { count, rows }; // return { count, rows };
}; };
@ -110,6 +110,17 @@ class Heytrip {
return ret; return ret;
}; };
/**
* ************************************************************************************************************
* 同步heytrip的酒店
* 1. 获取可用的酒店id, 缺少的设为[99=下架]
* 2. 根据ID获取酒店详情, 无信息设为下架
*/
/**
* @deprecated
* 第一次同步录库, 完成
*/
syncAids = async () => { syncAids = async () => {
const lastPageIndex = await this.getLastPageIndex(); const lastPageIndex = await this.getLastPageIndex();
const pageIndex = lastPageIndex + 1; const pageIndex = lastPageIndex + 1;
@ -131,6 +142,9 @@ class Heytrip {
}; };
}; };
/**
* 获取酒店ID
*/
syncAidState = async () => { syncAidState = async () => {
const today = new Date(); const today = new Date();
today.setHours(0, 0, 0, 0); // set the time to 00:00:00.000 today.setHours(0, 0, 0, 0); // set the time to 00:00:00.000
@ -140,7 +154,7 @@ class Heytrip {
pageIndex = lastPageIndex; pageIndex = lastPageIndex;
if (isEmpty(lastPageIndex)) { if (isEmpty(lastPageIndex)) {
lastPageIndex = await this.getLastPageIndex({ last_modify_time: { [Op.gt]: today }, page_index: { [Op.lt]: 9999 } }); lastPageIndex = await this.getLastPageIndex({ last_modify_time: { [Op.gt]: today }, page_index: { [Op.lt]: 9999 } });
console.log('syncAidState', 'lastPageIndex', lastPageIndex); // console.log('syncAidState', 'lastPageIndex', lastPageIndex);
pageIndex = lastPageIndex + 1; pageIndex = lastPageIndex + 1;
} }
@ -187,11 +201,13 @@ class Heytrip {
if (!isEmpty(stateNormal)) { if (!isEmpty(stateNormal)) {
await HeytripIds.update({ update_flag: 0, page_index: pageIndex, last_modify_time: Sequelize.fn('NOW') }, { where: { hotel_id: stateNormal } }); await HeytripIds.update({ update_flag: 0, page_index: pageIndex, last_modify_time: Sequelize.fn('NOW') }, { where: { hotel_id: stateNormal } });
} }
// 新增ID
const newIds = validIds.filter((id) => !savedIds.map((item) => item.hotel_id).includes(id)); const newIds = validIds.filter((id) => !savedIds.map((item) => item.hotel_id).includes(id));
if (!isEmpty(newIds)) { if (!isEmpty(newIds)) {
const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1 })); const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1, priority: -10 }));
await HeytripIds.bulkCreate(insertRows); await HeytripIds.bulkCreate(insertRows);
} }
// 页码滚动
const oldToNext = savedPageIds.filter((item) => !validIds.includes((item.hotel_id))).map((item) => item.hotel_id); const oldToNext = savedPageIds.filter((item) => !validIds.includes((item.hotel_id))).map((item) => item.hotel_id);
if (!isEmpty(oldToNext)) { if (!isEmpty(oldToNext)) {
await HeytripIds.update({ page_index: Number(pageIndex)+9999, update_flag: 99 }, { where: { hotel_id: oldToNext } }); await HeytripIds.update({ page_index: Number(pageIndex)+9999, update_flag: 99 }, { where: { hotel_id: oldToNext } });
@ -203,13 +219,29 @@ class Heytrip {
}; };
} }
newHotels = async () => { /**
* 获取新添加的酒店
* 在`syncAidState`中设置了`update_flag=1`, `priority=-10`
*/
newHotels = async (lgc) => {
const [rows] = await Sequelize.query( const [rows] = await Sequelize.query(
'SELECT i.hotel_id, IFNULL(h.hi_sn ,0) info_exists FROM heytrip_ids as i LEFT JOIN hotelinfo AS h ON h.hotel_id = i.hotel_id WHERE h.hi_sn IS NULL AND update_flag != 0 ORDER BY info_exists LIMIT 10' `SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists
FROM heytrip_ids AS i
LEFT JOIN hotelinfo2 AS h ON h.hotel_id = i.hotel_id
AND h.lgc = ${lgc}
WHERE h.hi2_sn IS NULL
AND update_flag = 1
ORDER BY info_exists LIMIT 10`
, { logging: false }
); );
return rows; const res = await this.syncInitHotelLgcDetailsAction(rows, LGC_MAPPED[lgc]);
return res;
}; };
/**
* 获取缺少的语种详情
*/
newHotelsLgc = async (lgc) => { newHotelsLgc = async (lgc) => {
const [rows] = await Sequelize.query( const [rows] = await Sequelize.query(
`SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists `SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists
@ -233,6 +265,11 @@ class Heytrip {
const res = await this.syncInitHotelDetailsAction(rows, LGC_MAPPED['2']); const res = await this.syncInitHotelDetailsAction(rows, LGC_MAPPED['2']);
return res; return res;
}; };
/**
* 中国酒店: 获取缺少的语种详情
* * 中国: id > 20000000
*/
chinaHotelsLgc2 = async (lgc) => { chinaHotelsLgc2 = async (lgc) => {
const [rows] = await Sequelize.query( const [rows] = await Sequelize.query(
`SELECT i.hotel_id `SELECT i.hotel_id
@ -242,6 +279,7 @@ class Heytrip {
AND h2.lgc = ${lgc} AND h2.lgc = ${lgc}
WHERE h2.hi2_sn IS NULL WHERE h2.hi2_sn IS NULL
AND i.hotel_id > 20000000 AND i.hotel_id > 20000000
AND i.update_flag != 99
LIMIT 10` LIMIT 10`
, { logging: false } , { logging: false }
); );
@ -263,6 +301,7 @@ class Heytrip {
/** /**
* @deprecated * @deprecated
* 第一次录库, 已执行
*/ */
syncInitHotelDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => { syncInitHotelDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => {
let allIds = []; let allIds = [];
@ -309,11 +348,13 @@ class Heytrip {
} }
}; };
/**
* 录入酒店的信息
* todo: 更新详情
*/
syncInitHotelLgcDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => { syncInitHotelLgcDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => {
let allIds = [];
try { try {
allIds = rows.map((item) => item.hotel_id); const allIds = rows.map((item) => item.hotel_id);
console.log('allIds', allIds);
if (isEmpty(rows)) { if (isEmpty(rows)) {
return { next: !isEmpty(allIds), data: allIds }; return { next: !isEmpty(allIds), data: allIds };
@ -321,7 +362,6 @@ class Heytrip {
const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds }, }); const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds }, });
const _BaseInfoExistsMapped = _BaseInfoExists.reduce((ru, c) => ({...ru, [`${c.hotel_id}`]: c }), {}); const _BaseInfoExistsMapped = _BaseInfoExists.reduce((ru, c) => ({...ru, [`${c.hotel_id}`]: c }), {});
const existsIds = _BaseInfoExists.map((item) => `${item.hotel_id}`); const existsIds = _BaseInfoExists.map((item) => `${item.hotel_id}`);
// console.log('existsIds', existsIds);
const res = await AccommodationsDetails({ const res = await AccommodationsDetails({
Language: lgcObj.locale, Language: lgcObj.locale,
@ -331,22 +371,30 @@ class Heytrip {
// hotel info // hotel info
const insertData = resolveDetails(res, lgcObj); const insertData = resolveDetails(res, lgcObj);
// return insertData; // debug: 0 // return insertData; // debug: 0
const resIds = insertData.info.map((item) => item.hotel_id); const resIds = insertData.info.map((item) => `${item.hotel_id}`);
/** 开始Database */ /** 开始Database */
const result = await Sequelize.transaction(async (transaction) => { const result = await Sequelize.transaction(async (transaction) => {
const sequelizeOptions = { logging: false, transaction }; const sequelizeOptions = { logging: false, transaction };
let Info; let Info;
/**
* 无返回数据, 设为失效`99`
* * 但部分中国酒店, 只有中文数据, 请求英文无返回
*/
const offInfo = allIds.filter((iitem) => !resIds.includes(`${iitem}`)); const offInfo = allIds.filter((iitem) => !resIds.includes(`${iitem}`));
if (!isEmpty(offInfo)) { if (!isEmpty(offInfo)) {
await HeytripIds.update({ update_flag: 99 }, { where: { hotel_id: offInfo }, ...sequelizeOptions }); const flag = Number(lgcObj.lgc) === 1 ? 2 : 99; // 等待获取中文数据
const priority = Number(lgcObj.lgc) === 1 ? 0 : 99;
await HeytripIds.update({ update_flag: flag, priority }, { where: { hotel_id: offInfo }, ...sequelizeOptions });
} }
const updateInfo = insertData.info.filter((iitem) => existsIds.includes(`${iitem.hotel_id}`)); const updateInfo = insertData.info.filter((iitem) => existsIds.includes(`${iitem.hotel_id}`));
if (!isEmpty(updateInfo)) { if (!isEmpty(updateInfo)) {
await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
for await (const updateRow of updateInfo) { for await (const updateRow of updateInfo) {
// 有中英文的, 把名称合并, eg.上海意家人酒店Yijiaren Hotel
const _BaseName = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].hotel_name; const _BaseName = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].hotel_name;
if ((_BaseName || '').includes((updateRow.hotel_name || '').substring(0, 4))) { if ((_BaseName || '').includes((updateRow.hotel_name || '').substring(0, 4))) {
continue; continue;
@ -354,9 +402,9 @@ class Heytrip {
const _BaseAddress = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].address; const _BaseAddress = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].address;
await Hotelinfo.update( await Hotelinfo.update(
{ {
update_flag: 0, update_flag: 0, priority: 0,
hotel_name: lgcObj.lgc === 1 ? `${_BaseName}${updateRow.hotel_name}` : `${updateRow.hotel_name}${_BaseName}`, hotel_name: Number(lgcObj.lgc) === 1 ? `${_BaseName}${updateRow.hotel_name}` : `${updateRow.hotel_name}${_BaseName}`,
address: lgcObj.lgc === 1 ? `${_BaseAddress}${updateRow.address}` : `${updateRow.address}${_BaseAddress}`, address: Number(lgcObj.lgc) === 1 ? `${_BaseAddress}${updateRow.address}` : `${updateRow.address}${_BaseAddress}`,
}, },
{ where: { hotel_id: updateRow.hotel_id }, ...sequelizeOptions } { where: { hotel_id: updateRow.hotel_id }, ...sequelizeOptions }
); );
@ -378,7 +426,8 @@ class Heytrip {
if (!isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions); if (!isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions);
if (!isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions); if (!isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions);
if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions }); if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
if (!isEmpty(updateInfo)) await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
return Info; return Info;
}); });
@ -387,10 +436,13 @@ class Heytrip {
} catch (error) { } catch (error) {
console.log(error); console.log(error);
return { next: false, restart: true, data: allIds }; return { next: false, restart: true, };
} }
}; };
/**
* 获取实时的报价
*/
getHotelAvailability = async (param) => { getHotelAvailability = async (param) => {
const { hotel_id, checkin, checkout, adults, children_ages, rooms, nationality } = param; const { hotel_id, checkin, checkout, adults, children_ages, rooms, nationality } = param;
const paramBody = { const paramBody = {
@ -427,4 +479,5 @@ class Heytrip {
return await Logs.create(data, { logging: false }); return await Logs.create(data, { logging: false });
}; };
} }
module.exports = new Heytrip(); module.exports = new Heytrip();

Write Preview
Loading…
Cancel
Save