Haina
/
heytripgo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 同步: 同步ID, 更新下架状态; 新增的获取数据

Browse Source
main
Lei OT 10 months ago
parent a51e96f866
commit a0cdcc28c1
4 changed files with 200 additions and 65 deletions
Show Stats Download Patch File Download Diff File

9
server/app.js
Unescape Escape View File

@ -9,7 +9,7 @@ const logger = require('koa-logger')
const index = require('./routes/index')
const { Aids: syncAids, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs');
const { Aids: syncAids, AidsState: syncAidsState, hotelLgcDetails: syncHotelDetails, chinaHotelDetails: syncChinas } = require('./jobs/syncHeytripJobs');
const rlog = require('./middleware/request_log');
// error handler
@ -41,9 +41,10 @@ app.use(async (ctx, next) => {
})
// schedule jobs
// syncAids();
// syncHotelDetails();
// syncChinas();
syncAids();
syncAidsState();
syncHotelDetails();
syncChinas();
// routes
app.use(index.routes(), index.allowedMethods())

55
server/jobs/syncHeytripJobs.js
Unescape Escape View File

@ -1,29 +1,63 @@
const { scheduleJob } = require('node-schedule');
const heytripService = require('../services/heytripService');
const { LGC_MAPPED } = require('../config/constants');
/**
* 获取可用的酒店id
*/
const Aids = () => {
const job = scheduleJob('*/2 * * * * *', async function () {
return false;
// const jobA = scheduleJob('*/2 * * * * *', async function () {
const jobA = scheduleJob('0 0 0 * * *', async function () {
console.log('syncing heytrip, get available accommodation ids.');
const isRunning = jobA.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.syncAids();
if (res.nextPage !== true) {
job.cancel();
console.log('job completed! canceled job!');
jobA.cancel();
}
} else {
console.log('pre job running! cancelNext');
jobA.cancelNext();
}
});
};
/**
* 更新酒店的状态, 是否下架
*/
const AidsState = () => {
// const jobAS = scheduleJob('*/2 * * * * *', async function () {
const jobAS = scheduleJob('0 5 0 * * *', async function () {
console.log('--------------------syncing heytrip, get available accommodation ids.--------------------');
const isRunning = jobAS.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
const res = await heytripService.syncAidState();
if (res.nextPage !== true) {
console.log('job completed! canceled job!');
jobAS.cancel();
}
} else {
console.log('pre job running! cancelNext');
jobAS.cancelNext();
}
});
};
/**
* 更新酒店详情, 按语种
*/
const hotelLgcDetails = () => {
// return false;
const job2 = scheduleJob('*/4 * * * * *', async function () {
console.log('-------------------------syncing heytrip, get accommodation details.-------------------------');
const isRunning = job2.pendingInvocations[0]?.job?.running == 1;
if (!isRunning) {
// const res = await heytripService.syncHotelDetails();
// const res = await heytripService.syncInitHotelLgcDetailsAction(LGC_MAPPED['1']);
const res = await heytripService.newHotelsLgc('1');
job2.cancel(); // debug: 0
// job2.cancel(); // debug: 0
if (res.next !== true) {
job2.cancel();
console.log('job completed! canceled job!');
@ -35,7 +69,11 @@ const hotelLgcDetails = () => {
});
};
/**
* 更新中国酒店详情. 已完成
*/
const chinaHotelDetails = () => {
return false;
const job3 = scheduleJob('*/4 * * * * *', async function () {
console.log('syncing heytrip, get china accommodation details.');
const isRunning = job3.pendingInvocations[0]?.job?.running == 1;
@ -54,5 +92,6 @@ const chinaHotelDetails = () => {
};
module.exports = {
Aids, hotelLgcDetails, chinaHotelDetails
Aids, AidsState,
hotelLgcDetails, chinaHotelDetails
}

155
server/services/heytripService.js
Unescape Escape View File

@ -3,7 +3,7 @@ const initModels = require('./../models/init-models');
const { AvailableAccommodationIds, AccommodationsDetails, Availability } = require('../vendor/heytrip');
const { isEmpty } = require('../utils/commons');
const { resolveDetails, resolveRatePlans } = require('../helper/heytripDataHelper');
const { DEFAULT_LGC, LGC_MAPPED } = require('../config/constants');
const { DEFAULT_LGC, LGC_MAPPED, HEYTRIP_API_PROD } = require('../config/constants');
const { sequelize: Sequelize, Op } = db;
@ -24,6 +24,9 @@ const ReviewsSummaries = models.reviewsSummariesModel;
const Locations = models.locationsModel;
// const CacheAvailability = models.cacheAvailabilityModel;
const Logs = models.requestLogsModel;
// HeytripIds.hasMany(Hotelinfo, {
// foreignKey: 'hotel_id',
// onDelete: 'NO ACTION',
@ -32,7 +35,8 @@ const Locations = models.locationsModel;
// Hotelinfo.belongsTo(HeytripIds, { as: 'aid', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' });
// Hotelinfo.hasMany(Rooms, { sourceKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' });
// Rooms.belongsTo(Hotelinfo, { targetKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', });
Hotelinfo.hasMany(Hotelinfo2, { as: 'lgc_info', sourceKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' });
Hotelinfo.belongsTo(HeytripIds, { as: 'aid', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' });
Hotelinfo.hasMany(Hotelinfo2, { as: 'locale_info', sourceKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION' });
Hotelinfo.hasMany(City, { as: 'city', sourceKey: 'city_id', foreignKey: 'id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', }); // 多语种, 所以实际是 hasMany , 用 hasOne 要指定 lgc= 1 或者2
Hotelinfo.hasMany(Country, { as: 'country', sourceKey: 'country_code', foreignKey: 'id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', }); // 多语种, 所以实际是 hasMany , 用 hasOne 要指定 lgc= 1 或者2
// Hotelinfo2.belongsTo(Hotelinfo, { targetKey: 'hotel_id', foreignKey: 'hotel_id', onDelete: 'NO ACTION', onUpdate: 'NO ACTION', });
@ -45,8 +49,8 @@ class Heytrip {
const keywordSplit = keyword.split(' ');
const keywordWhere = (field) => keywordSplit.map((word) => Sequelize.where(Sequelize.fn('instr', Sequelize.col(field), word), { [Op.gt]: 0 }));
// 'hotelinfo.address'
const keywordSearch = ['hotelinfo.hotel_name', ].map((field) => ({ [Op.and]: keywordWhere(field) }));
const keywordOrder = ['hotelinfo.hotel_name',].reduce((ro, field) => ro.concat(keywordSplit.map((word) => Sequelize.fn('instr', Sequelize.col(field), word))), []);
const keywordSearch = ['hotelinfo.hotel_name'].map((field) => ({ [Op.and]: keywordWhere(field) }));
const keywordOrder = ['hotelinfo.hotel_name'].reduce((ro, field) => ro.concat(keywordSplit.map((word) => Sequelize.fn('instr', Sequelize.col(field), word))), []);
// const keywordSearchCount = ['hi2_hotel_name', 'hi2_address'].map((field) => ({ [Op.and]: keywordWhere(field) }));
// const countRows = await Hotelinfo2.count({
@ -62,9 +66,10 @@ class Heytrip {
const { count, rows } = await Hotelinfo.findAndCountAll({
// const [ rows ] = await Hotelinfo.findAll({
include: [
{ model: HeytripIds, as: 'aid', attributes: ['hotel_id', 'update_flag'], where: { update_flag: { [Op.ne]: 99 } } },
{
model: Hotelinfo2,
as: 'lgc_info',
as: 'locale_info',
attributes: [
['hi2_sn', 'sn'],
'hotel_id',
@ -79,8 +84,8 @@ class Heytrip {
required: false,
separate: true,
},
{ model: City, as: 'city', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true, },
{ model: Country, as: 'country', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true, },
{ model: City, as: 'city', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true },
{ model: Country, as: 'country', attributes: ['id', 'name'], where: { lgc: 2 }, required: false, separate: true },
],
// include: [{ model: Hotelinfo2, as: 'h2', required: true }],
where: {
@ -97,8 +102,12 @@ class Heytrip {
// return { count, rows };
};
getLastPageIndex = async () => {
const ret = await HeytripIds.max('page_index');
getLastPageIndex = async (where = {}) => {
const ret = await HeytripIds.max('page_index', { where });
return ret;
};
getFirstPageIndex = async (where = {}) => {
const ret = await HeytripIds.min('page_index', { where });
return ret;
};
@ -123,6 +132,76 @@ class Heytrip {
};
};
syncAidState = async () => {
const today = new Date();
today.setHours(0, 0, 0, 0); // set the time to 00:00:00.000
let lastPageIndex; let pageIndex;
lastPageIndex = await this.getFirstPageIndex({ last_modify_time: { [Op.lt]: today } });
pageIndex = lastPageIndex;
if (isEmpty(lastPageIndex)) {
lastPageIndex = await this.getLastPageIndex({ last_modify_time: { [Op.gt]: today }, page_index: { [Op.lt]: 9999 } });
pageIndex = lastPageIndex + 1;
}
console.log('syncAidState', lastPageIndex, pageIndex);
const validIds = (await AvailableAccommodationIds(pageIndex)).map((id) => String(id));
if (isEmpty(validIds)) {
// await HeytripIds.update({ update_flag: 99, priority: 99 }, { where: { pageIndex: [Op.gt]: pageIndex } });
// 同步结束; 本次没有的ID: 更新: 99=失效
const stateOff = await HeytripIds.findAll({
raw: true,
where: {
update_flag: { [Op.notIn]: [0, 1] },
last_modify_time: { [Op.gt]: today },
},
attributes: ['hotel_id'],
});
const stateOffIds = stateOff.map((item) => item.hotel_id);
if (!isEmpty(stateOffIds)) {
await HeytripIds.update({ update_flag: 99, priority: 99 }, { where: { hotel_id: stateOffIds } });
console.log('updated stateOff', stateOffIds.length);
}
return {
nextPage: false,
};
}
const savedIds = await HeytripIds.findAll({
raw: true,
where: { hotel_id: validIds },
attributes: ['hotel_id'],
logging: false,
}); // savedIds <= validIds
const savedPageIds = await HeytripIds.findAll({
raw: true,
where: { page_index: pageIndex, },
attributes: ['hotel_id'],
// logging: false,
});
// 已存在ID: 更新: 状态, 页码, 时间
const stateNormal = savedIds.filter((item) => validIds.includes((item.hotel_id))).map((item) => item.hotel_id);
if (!isEmpty(stateNormal)) {
await HeytripIds.update({ update_flag: 0, page_index: pageIndex, last_modify_time: Sequelize.fn('NOW') }, { where: { hotel_id: stateNormal } });
}
const newIds = validIds.filter((id) => !savedIds.map((item) => item.hotel_id).includes(id));
if (!isEmpty(newIds)) {
const insertRows = newIds.map((id) => ({ hotel_id: id, page_index: pageIndex, update_flag: 1 }));
await HeytripIds.bulkCreate(insertRows);
}
const oldToNext = savedPageIds.filter((item) => !validIds.includes((item.hotel_id))).map((item) => item.hotel_id);
if (!isEmpty(oldToNext)) {
await HeytripIds.update({ page_index: Number(pageIndex)+9999, update_flag: 99 }, { where: { hotel_id: oldToNext } });
}
return {
nextPage: true,
pageIndex,
};
}
newHotels = async () => {
const [rows] = await Sequelize.query(
'SELECT i.hotel_id, IFNULL(h.hi_sn ,0) info_exists FROM heytrip_ids as i LEFT JOIN hotelinfo AS h ON h.hotel_id = i.hotel_id WHERE h.hi_sn IS NULL AND update_flag != 0 ORDER BY info_exists LIMIT 10'
@ -131,13 +210,6 @@ class Heytrip {
};
newHotelsLgc = async (lgc) => {
// const [rows] = await Sequelize.query(
// `SELECT h.hotel_id
// FROM hotelinfo AS h
// LEFT JOIN hotelinfo2 AS h2 ON h.hotel_id =h2.hotel_id
// AND h2.lgc = ${lgc}
// WHERE h2.hi2_sn IS NULL LIMIT 10`
// );
const [rows] = await Sequelize.query(
`SELECT i.hotel_id ,IFNULL(h.hi2_sn, 0) info_exists
FROM heytrip_ids AS i
@ -146,6 +218,7 @@ class Heytrip {
WHERE h.hi2_sn IS NULL
AND update_flag != 99
ORDER BY info_exists LIMIT 10`
, { logging: false }
);
const res = await this.syncInitHotelLgcDetailsAction(rows, LGC_MAPPED[lgc]);
return res;
@ -182,10 +255,12 @@ class Heytrip {
return res;
};
/**
* @deprecated
*/
syncInitHotelDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => {
let allIds = [];
try {
allIds = rows.map((item) => item.hotel_id);
const updateIds = rows.filter((item) => item.info_exists !== 0).map((item) => item.hotel_id);
const newIds = rows.filter((item) => item.info_exists === 0).map((item) => item.hotel_id);
@ -204,8 +279,8 @@ class Heytrip {
// return insertData; // debug: 0
/** 开始Database */
const sequelizeOptions = { logging: false };
const result = await Sequelize.transaction(async t => {
const result = await Sequelize.transaction(async (transaction) => {
const sequelizeOptions = { logging: false, transaction };
let Info;
if (!isEmpty(insertData.info)) Info = await Hotelinfo.bulkCreate(insertData.info, sequelizeOptions);
if (!isEmpty(insertData.info2)) await Hotelinfo2.bulkCreate(insertData.info2, sequelizeOptions);
@ -221,26 +296,23 @@ class Heytrip {
});
return { next: !isEmpty(allIds), data: allIds };
} catch (error) {
console.log(error);
return { next: false, data: allIds };
}
}
};
syncInitHotelLgcDetailsAction = async (rows, lgcObj = LGC_MAPPED[DEFAULT_LGC]) => {
let allIds = [];
try {
allIds = rows.map((item) => item.hotel_id);
if (isEmpty(rows)) {
return { next: !isEmpty(allIds), data: allIds };
}
const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds } });
const _BaseInfoExists = await Hotelinfo.findAll({ where: { hotel_id: allIds }, logging: false });
const updateIds = _BaseInfoExists.map((item) => `${item.hotel_id}`);
console.log('updateIds', updateIds);
// console.log('updateIds', updateIds);
const res = await AccommodationsDetails({
Language: lgcObj.locale,
@ -253,12 +325,20 @@ class Heytrip {
// return insertData; // debug: 0
/** 开始Database */
const result = await Sequelize.transaction(async transaction => {
const result = await Sequelize.transaction(async (transaction) => {
const sequelizeOptions = { logging: false, transaction };
let Info;
const newInfo = insertData.info.filter(iitem => !updateIds.includes(`${iitem.hotel_id}`));
console.log('newInfo', newInfo.map(xx => xx.hotel_id));
const offInfo = allIds.filter((iitem) => !resIds.includes(`${iitem}`));
if (!isEmpty(offInfo)) {
await HeytripIds.update({ update_flag: 99 }, { where: { hotel_id: offInfo }, ...sequelizeOptions });
}
const newInfo = insertData.info.filter((iitem) => !updateIds.includes(`${iitem.hotel_id}`));
// console.log(
// 'newInfo',
// newInfo.map((xx) => xx.hotel_id)
// );
if (!isEmpty(newInfo)) Info = await Hotelinfo.bulkCreate(newInfo, sequelizeOptions);
if (!isEmpty(insertData.info2)) await Hotelinfo2.bulkCreate(insertData.info2, sequelizeOptions);
@ -270,13 +350,12 @@ class Heytrip {
if (!isEmpty(insertData.reviews_summaries)) await ReviewsSummaries.bulkCreate(insertData.reviews_summaries, sequelizeOptions);
if (!isEmpty(insertData.locations)) await Locations.bulkCreate(insertData.locations, sequelizeOptions);
if ( ! isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: newInfo.map(x => x.hotel_id) } });
if (!isEmpty(newInfo)) await HeytripIds.update({ update_flag: 0 }, { where: { hotel_id: newInfo.map((x) => x.hotel_id) }, ...sequelizeOptions });
return Info;
});
return { next: !isEmpty(allIds), data: allIds };
} catch (error) {
console.log(error);
@ -303,5 +382,21 @@ class Heytrip {
const quoteRes = resolveRatePlans(_quoteRes);
return quoteRes;
};
writeHeytripRequestLog = async ({ action, body }) => {
const actionMapped = {
'getHotelAvailability': '/Accommodation/Availability',
'getHotelInfo': '/Accommodation/AccommodationsDetails',
'getHotelIds': '/Accommodation/AvailableAccommodationIds',
};
const data = {
action: action,
// method: ctx.method,
path: actionMapped[action],
request_data: body,
// ip: ctx.ip,
};
return await Logs.create(data, { logging: false });
};
}
module.exports = new Heytrip();

2
server/vendor/heytrip.js vendored
Unescape Escape View File

@ -23,7 +23,7 @@ const AvailableAccommodationIds = async (pageIndex) => {
},
});
console.log('Call pageIndex', pageIndex, response.data.TotalPage);
return response.data.Data;
return response.data.Data || [];
};
const AccommodationsDetails = async (body) => {

Write Preview
Loading…
Cancel
Save