conf: 同步触发规则

main
Lei OT 10 months ago
parent 997e9adbeb
commit 773ef97a27

@ -11,7 +11,7 @@ const log4js = require('./config/log4');
const index = require('./routes/index')
const { Aids: syncAids, AidsState: syncAidsState, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs');
const { Aids: syncAids, AidsState: syncAidsState, newHotelDetails: syncNewHotels, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs');
const rlog = require('./middleware/request_log');
// error handler
@ -43,8 +43,9 @@ app.use(async (ctx, next) => {
})
// schedule jobs
syncAids();
// syncAids();
syncAidsState();
syncNewHotels();
syncHotelDetails();
syncChinas();

@ -1,98 +1,151 @@
const { scheduleJob } = require('node-schedule');
const heytripService = require('../services/heytripService');
/**
* node-schedule
* ? * * * * * *
day of week (0 - 7) (0 or 7 is Sun)
month (1 - 12)
day of month (1 - 31)
hour (0 - 23)
minute (0 - 59)
second (0 - 59, OPTIONAL)
*/
/**
* 获取可用的酒店id
*/
const Aids = () => {
return false;
// const jobA = scheduleJob('*/2 * * * * *', async function () {
const jobA = scheduleJob('0 0 0 * * *', async function () {
console.log('syncing heytrip, get available accommodation ids.');
const isRunning = jobA.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.syncAids();
if (res.nextPage !== true) {
console.log('job completed! canceled job!');
jobA.cancel();
}
} else {
console.log('pre job running! cancelNext');
jobA.cancelNext();
}
// const jobA = scheduleJob('0 0 0 * * *', async function () {
// console.log('syncing heytrip, get available accommodation ids.');
// const isRunning = jobA.pendingInvocations[0]?.job?.running == 1;
// if (!isRunning) {
// const res = await heytripService.syncAids();
// if (res.nextPage !== true) {
// console.log('job completed! canceled job!');
// jobA.cancel();
// }
// } else {
// console.log('pre job running! cancelNext');
// jobA.cancelNext();
// }
});
// });
};
/**
* 更新酒店的状态, 是否下架
* 每天启动同步;
* * 每次同步需要3000+
*/
const AidsState = () => {
const jobAS = scheduleJob('*/2 * * * * *', async function () {
// const jobAS = scheduleJob('0 5 0 * * *', async function () {
console.log('--------------------syncing heytrip, get available accommodation ids.--------------------');
const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.syncAidState();
// jobAS.cancel(); // debug: 0
if (res.nextPage !== true) {
console.log('job completed! canceled job[AidsState]!');
jobAS.cancel();
const dailyJob = scheduleJob('0 0 0 * * *', async function () {
const jobAS = scheduleJob('*/2 * * * * *', async function () {
console.log('--------------------syncing heytrip, get available accommodation ids.--------------------');
const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.syncAidState();
// jobAS.cancel(); // debug: 0
if (res.nextPage !== true) {
console.log('job completed! canceled job[AidsState]!');
jobAS.cancel();
}
} else {
console.log('pre job running! cancelNext[AidsState]');
jobAS.cancelNext();
}
});
});
};
/**
* 获取新增的酒店详情
* ID同步结束后启动, ID大约需要3小时
* 每天03:05:00启动
*/
const newHotelDetails = () => {
// return false;
const dailyJob1 = scheduleJob('0 5 3 * * *', async function () {
const job1 = scheduleJob('*/5 * * * * *', async function () {
console.log('-------------------------syncing heytrip, get accommodation details.-------------------------');
const isRunning = job1.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.newHotels('1');
// job1.cancel(); // debug: 0
if (res.next !== true) {
job1.cancel(res.restart);
console.log('job completed! canceled job[newHotelDetails]!');
}
} else {
console.log('pre job running! cancelNext[newHotelDetails]');
job1.cancelNext();
}
} else {
console.log('pre job running! cancelNext[AidsState]');
jobAS.cancelNext();
}
});
});
};
/**
* 更新酒店详情, 按语种
* 每周启动一次
* 周五20:05:00启动
*/
const hotelLgcDetails = () => {
// return false;
const job2 = scheduleJob('*/4 * * * * *', async function () {
console.log('-------------------------syncing heytrip, get accommodation details.-------------------------');
const dailyJob2 = scheduleJob('0 5 20 * * 5', async function () {
const job2 = scheduleJob('*/5 * * * * *', async function () {
console.log('-------------------------syncing heytrip, get accommodation details.-------------------------');
const isRunning = job2.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.newHotelsLgc('1');
const isRunning = job2.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.newHotelsLgc('1');
// job2.cancel(); // debug: 0
if (res.next !== true) {
job2.cancel();
console.log('job completed! canceled job[hotelLgcDetails]!');
// job2.cancel(); // debug: 0
if (res.next !== true) {
job2.cancel(res.restart);
console.log('job completed! canceled job[hotelLgcDetails]!');
}
} else {
console.log('pre job running! cancelNext[hotelLgcDetails]');
job2.cancelNext();
}
} else {
console.log('pre job running! cancelNext[hotelLgcDetails]');
job2.cancelNext();
}
});
});
};
/**
* 更新中国酒店详情. 已完成
* 更新中国酒店的中文详情.
* 每周启动一次
* 周六20:05:00启动
*/
const chinaHotelDetails = () => {
return false;
const job3 = scheduleJob('*/4 * * * * *', async function () {
console.log('syncing heytrip, get china accommodation details.');
const isRunning = job3.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.chinaHotelsLgc2('2');
if (res.next !== true) {
job3.cancel();
console.log('job completed! canceled job!');
// job3.reschedule('0 0 0 * * *');
// return false;
const dailyJobCN = scheduleJob('0 5 20 * * 6', async function () {
const job3 = scheduleJob('*/4 * * * * *', async function () {
console.log('-------------------------syncing heytrip, get china accommodation details.-------------------------');
const isRunning = job3.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.chinaHotelsLgc2('2');
// job3.cancel(); // debug: 0
if (res.next !== true) {
job3.cancel(res.restart);
console.log('job completed! canceled job[chinaHotelDetails]!');
// job3.reschedule('0 0 0 * * *');
}
} else {
console.log('pre job running! cancelNext');
job3.cancelNext();
}
} else {
console.log('pre job running! cancelNext');
job2.cancelNext();
}
});
});
};
module.exports = {
Aids, AidsState,
hotelLgcDetails, chinaHotelDetails
newHotelDetails, hotelLgcDetails, chinaHotelDetails
}

@ -97,7 +97,7 @@ class Heytrip {
// nest: true,
});
return { count, rows: rows.map((item) => item.dataValues) };
return { count, rows: rows.map((item) => item.toJSON()) };
// return { count, rows };
};
@ -110,6 +110,17 @@ class Heytrip {
return ret;
};
/**
* ************************************************************************************************************
* 同步heytrip的酒店
* 1. 获取可用的酒店id, 缺少的设为[99=下架]
* 2. 根据ID获取酒店详情, 无信息设为下架
*/
/**
* @deprecated
* 第一次同步录库, 完成
*/
syncAids = async () => {
const lastPageIndex = await this.getLastPageIndex();
const pageIndex = lastPageIndex + 1;
@ -131,6 +142,9 @@ class Heytrip {
};
};
/**
* 获取酒店ID
*/
syncAidState = async () => {
const today = new Date();
today.setHours(0, 0, 0, 0); // set the time to 00:00:00.000
@ -140,7 +154,7 @@ class Heytrip {
pageIndex = lastPageIndex;
if (isEmpty(lastPageIndex)) {
lastPageIndex = await this.getLastPageIndex({ last_modify_time: { [Op.gt]: today }, page_index: { [Op.lt]: 9999 } });
console.log('syncAidState', 'lastPageIndex', lastPageIndex);
// console.log('syncAidState', 'lastPageIndex', lastPageIndex);
pageIndex = lastPageIndex + 1;
}
@ -187,11 +201,13 @@ class Heytrip {
if (!isEmpty(stateNormal)) {
await HeytripIds.update({ update_flag: 0, page_index: pageIndex, last_modify_time: Sequelize.fn('NOW') }, { where: { hotel_id: stateNormal } });
}
// 新增ID
const newIds = validIds.filter((id) => !savedIds.map((item) => item.hotel_id).includes(id));
if (!isEmpty(newIds)) {
const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1 }));
const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1, priority: -10 }));
await HeytripIds.bulkCreate(insertRows);
}
// 页码滚动
const oldToNext = savedPageIds.filter((item) => !validIds.includes((item.hotel_id))).map((item) => item.hotel_id);
if (!isEmpty(oldToNext)) {
await HeytripIds.update({ page_index: Number(pageIndex)+9999, update_flag: 99 }, { where: { hotel_id: oldToNext } });
@ -203,13 +219,29 @@ class Heytrip {
};
}
newHotels = async () => {
/**
* 获取新添加的酒店
* `syncAidState`中设置了`update_flag=1`, `priority=-10`
*/
newHotels = async (lgc) => {
const [rows] = await Sequelize.query(
'SELECT i.hotel_id, IFNULL(h.hi_sn ,0) info_exists FROM heytrip_ids as i LEFT JOIN hotelinfo AS h ON h.hotel_id = i.hotel_id WHERE h.hi_sn IS NULL AND update_flag != 0 ORDER BY info_exists LIMIT 10'
`SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists
FROM heytrip_ids AS i
LEFT JOIN hotelinfo2 AS h ON h.hotel_id = i.hotel_id
AND h.lgc = ${lgc}
WHERE h.hi2_sn IS NULL
AND update_flag = 1
ORDER BY info_exists LIMIT 10`
, { logging: false }
);
return rows;
const res = await this.syncInitHotelLgcDetailsAction(rows, LGC_MAPPED[lgc]);
return res;
};
/**
* 获取缺少的语种详情
*/
newHotelsLgc = async (lgc) => {
const [rows] = await Sequelize.query(
`SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists
@ -233,15 +265,21 @@ class Heytrip {
const res = await this.syncInitHotelDetailsAction(rows, LGC_MAPPED['2']);
return res;
};
/**
* 中国酒店: 获取缺少的语种详情
* * 中国: id > 20000000
*/
chinaHotelsLgc2 = async (lgc) => {
const [rows] = await Sequelize.query(
`SELECT i.hotel_id
-- FROM hotelinfo AS h
FROM heytrip_ids AS i
LEFT JOIN hotelinfo2 AS h2 ON i.hotel_id =h2.hotel_id
AND h2.lgc = ${lgc}
AND h2.lgc = ${lgc}
WHERE h2.hi2_sn IS NULL
AND i.hotel_id > 20000000
AND i.hotel_id > 20000000
AND i.update_flag != 99
LIMIT 10`
, { logging: false }
);
@ -263,6 +301,7 @@ class Heytrip {
/**
* @deprecated
* 第一次录库, 已执行
*/
syncInitHotelDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => {
let allIds = [];
@ -309,11 +348,13 @@ class Heytrip {
}
};
/**
* 录入酒店的信息
* todo: 更新详情
*/
syncInitHotelLgcDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => {
let allIds = [];
try {
allIds = rows.map((item) => item.hotel_id);
console.log('allIds', allIds);
const allIds = rows.map((item) => item.hotel_id);
if (isEmpty(rows)) {
return { next: !isEmpty(allIds), data: allIds };
@ -321,7 +362,6 @@ class Heytrip {
const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds }, });
const _BaseInfoExistsMapped = _BaseInfoExists.reduce((ru, c) => ({...ru, [`${c.hotel_id}`]: c }), {});
const existsIds = _BaseInfoExists.map((item) => `${item.hotel_id}`);
// console.log('existsIds', existsIds);
const res = await AccommodationsDetails({
Language: lgcObj.locale,
@ -331,22 +371,30 @@ class Heytrip {
// hotel info
const insertData = resolveDetails(res, lgcObj);
// return insertData; // debug: 0
const resIds = insertData.info.map((item) => item.hotel_id);
const resIds = insertData.info.map((item) => `${item.hotel_id}`);
/** 开始Database */
const result = await Sequelize.transaction(async (transaction) => {
const sequelizeOptions = { logging: false, transaction };
let Info;
/**
* 无返回数据, 设为失效`99`
* * 但部分中国酒店, 只有中文数据, 请求英文无返回
*/
const offInfo = allIds.filter((iitem) => !resIds.includes(`${iitem}`));
if (!isEmpty(offInfo)) {
await HeytripIds.update({ update_flag: 99 }, { where: { hotel_id: offInfo }, ...sequelizeOptions });
const flag = Number(lgcObj.lgc) === 1 ? 2 : 99; // 等待获取中文数据
const priority = Number(lgcObj.lgc) === 1 ? 0 : 99;
await HeytripIds.update({ update_flag: flag, priority }, { where: { hotel_id: offInfo }, ...sequelizeOptions });
}
const updateInfo = insertData.info.filter((iitem) => existsIds.includes(`${iitem.hotel_id}`));
if (!isEmpty(updateInfo)) {
await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
for await (const updateRow of updateInfo) {
// 有中英文的, 把名称合并, eg.上海意家人酒店Yijiaren Hotel
const _BaseName = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].hotel_name;
if ((_BaseName || '').includes((updateRow.hotel_name || '').substring(0, 4))) {
continue;
@ -354,9 +402,9 @@ class Heytrip {
const _BaseAddress = _BaseInfoExistsMapped[`${updateRow.hotel_id}`].address;
await Hotelinfo.update(
{
update_flag: 0,
hotel_name: lgcObj.lgc === 1 ? `${_BaseName}${updateRow.hotel_name}` : `${updateRow.hotel_name}${_BaseName}`,
address: lgcObj.lgc === 1 ? `${_BaseAddress}${updateRow.address}` : `${updateRow.address}${_BaseAddress}`,
update_flag: 0, priority: 0,
hotel_name: Number(lgcObj.lgc) === 1 ? `${_BaseName}${updateRow.hotel_name}` : `${updateRow.hotel_name}${_BaseName}`,
address: Number(lgcObj.lgc) === 1 ? `${_BaseAddress}${updateRow.address}` : `${updateRow.address}${_BaseAddress}`,
},
{ where: { hotel_id: updateRow.hotel_id }, ...sequelizeOptions }
);
@ -378,7 +426,8 @@ class Heytrip {
if (!isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions);
if (!isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions);
if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
if (!isEmpty(updateInfo)) await HeytripIds.update({ update_flag: 0, priority: 0 }, { where: { hotel_id: updateInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
return Info;
});
@ -387,10 +436,13 @@ class Heytrip {
} catch (error) {
console.log(error);
return { next: false, restart: true, data: allIds };
return { next: false, restart: true, };
}
};
/**
* 获取实时的报价
*/
getHotelAvailability = async (param) => {
const { hotel_id, checkin, checkout, adults, children_ages, rooms, nationality } = param;
const paramBody = {
@ -427,4 +479,5 @@ class Heytrip {
return await Logs.create(data, { logging: false });
};
}
module.exports = new Heytrip();

Loading…
Cancel
Save