From 1ad5056ba79288d6971c5f5e61d39c62ebdd4985 Mon Sep 17 00:00:00 2001 From: flifloo Date: Sun, 22 Nov 2020 16:48:06 +0100 Subject: [PATCH 1/4] Add DB models --- .gitignore | 4 ++++ config/config_example.json | 1 + models/UE.js | 23 +++++++++++++++++++++++ models/event.js | 36 ++++++++++++++++++++++++++++++++++++ models/group.js | 23 +++++++++++++++++++++++ models/semester.js | 28 ++++++++++++++++++++++++++++ models/user.js | 3 +++ 7 files changed, 118 insertions(+) create mode 100644 models/UE.js create mode 100644 models/event.js create mode 100644 models/group.js create mode 100644 models/semester.js diff --git a/.gitignore b/.gitignore index e073a39..05878a8 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,7 @@ config/config.json # Sass output public/stylesheets + +# Floobits +.floo +.flooignore diff --git a/config/config_example.json b/config/config_example.json index 6e93448..702ca17 100644 --- a/config/config_example.json +++ b/config/config_example.json @@ -18,6 +18,7 @@ "contact": "Your Contact ", "mailPath": "https://yourServer.com" }, + "edt": "https://adelb.univ-lyon1.fr/jsp/custom/modules/plannings/anonymous_cal.jsp?resources=XXXX&projectId=XXXX&calType=ical&firstDate=", "secret": "keyboard cat", "passwordPrivateKey": "ecc635295f200847b79299df48e15759" } diff --git a/models/UE.js b/models/UE.js new file mode 100644 index 0000000..acda65b --- /dev/null +++ b/models/UE.js @@ -0,0 +1,23 @@ +"use strict"; + +const { Model } = require("sequelize"); + +module.exports = (sequelize, DataTypes) => { + class UE extends Model { + static associate(models) { + UE.hasMany(models.Event); + UE.belongsToMany(models.User, {through: "UEUser"}); + } + } + UE.init({ + name : { + type: DataTypes.STRING, + allowNull: false, + unique: true + } + }, { + sequelize, + modelName: "UE", + }); + return UE; +}; diff --git a/models/event.js b/models/event.js new file mode 100644 index 0000000..8e0da6e --- /dev/null +++ b/models/event.js @@ -0,0 +1,36 @@ +"use strict"; + +const { Model } = require("sequelize"); + +module.exports = (sequelize, DataTypes) => { + class Event extends Model { + static associate(models) { + Event.belongsToMany(models.Semester, {through: "EventSemester"}); + Event.belongsToMany(models.Group, {through: "EventGroup"}); + Event.belongsToMany(models.User, {through: "EventTeacher"}); + Event.belongsTo(models.UE); + } + } + Event.init({ + name: { + type: DataTypes.STRING, + allowNull: false + }, + locations: { + type: DataTypes.STRING, + allowNull: false + }, + startDate: { + type: DataTypes.DATE, + allowNull: false + }, + endDate: { + type: DataTypes.DATE, + allowNull: false + } + }, { + sequelize, + modelName: "Event", + }); + return Event; +}; diff --git a/models/group.js b/models/group.js new file mode 100644 index 0000000..d9a2cf7 --- /dev/null +++ b/models/group.js @@ -0,0 +1,23 @@ +"use strict"; + +const { Model } = require("sequelize"); + +module.exports = (sequelize, DataTypes) => { + class Group extends Model { + static associate(models) { + Group.belongsTo(models.Semester); + Group.belongsToMany(models.User, {through: "UserGroup"}); + Group.belongsToMany(models.Event, {through: "EventGroup"}); + } + } + Group.init({ + number: { + type: DataTypes.STRING, + allowNull: false + } + }, { + sequelize, + modelName: "Group", + }); + return Group; +}; diff --git a/models/semester.js b/models/semester.js new file mode 100644 index 0000000..fac1dad --- /dev/null +++ b/models/semester.js @@ -0,0 +1,28 @@ +"use strict"; + +const { Model } = require("sequelize"); + +module.exports = (sequelize, DataTypes) => { + class Semester extends Model { + static associate(models) { + Semester.hasMany(models.Group); + Semester.belongsToMany(models.Event, {through: "EventSemester"}); + } + } + Semester.init({ + year: { + type: DataTypes.INTEGER, + allowNull: false, + unique: "semester" + }, + name : { + type: DataTypes.STRING, + allowNull: false, + unique: "semester" + } + }, { + sequelize, + modelName: "Semester", + }); + return Semester; +}; diff --git a/models/user.js b/models/user.js index 82657b0..5847790 100644 --- a/models/user.js +++ b/models/user.js @@ -18,6 +18,9 @@ module.exports = (sequelize, DataTypes) => { class User extends Model { static associate(models) { + User.belongsToMany(models.Group, {through: "UserGroup"}); + User.belongsToMany(models.Event, {through: "EventTeacher"}) + User.belongsToMany(models.UE, {through: "UEUser"}); } checkPassword(password) { From 84c932d3f93d4ad1e64574716e7e4e615c0f4117 Mon Sep 17 00:00:00 2001 From: flifloo Date: Tue, 24 Nov 2020 01:09:20 +0100 Subject: [PATCH 2/4] Add agenda worker setup --- agenda/index.js | 18 ++++++++++++++++++ bin/www | 1 + 2 files changed, 19 insertions(+) create mode 100644 agenda/index.js diff --git a/agenda/index.js b/agenda/index.js new file mode 100644 index 0000000..813cb12 --- /dev/null +++ b/agenda/index.js @@ -0,0 +1,18 @@ +const { Worker } = require("worker_threads"); +let nb = 0; + + +function startWorker(workerArgs) { + let worker = new Worker(...workerArgs); + worker.on("error", (err) => { + console.error(err); + nb--; + if (nb > 0) + startWorker(workerArgs); + }); +} + +module.exports = (app) => { + const workerArgs = ["./agenda/worker.js", {workerData: app.get("config")}]; + startWorker(workerArgs); +}; diff --git a/bin/www b/bin/www index e8cd115..0e0c83c 100755 --- a/bin/www +++ b/bin/www @@ -119,4 +119,5 @@ function onListening() { ? "pipe " + addr : "port " + addr.port; debug("Listening on " + bind); + require("../agenda")(app); } From 9de9cf7b37979b19831b530355a113bb0ad6f6dc Mon Sep 17 00:00:00 2001 From: flifloo Date: Tue, 24 Nov 2020 08:47:13 +0100 Subject: [PATCH 3/4] Socket getter for database --- sockets/get/agendaGet.js | 39 ++++++++++++++++++++++++++++++++++++++ sockets/get/groupGet.js | 21 ++++++++++++++++++++ sockets/get/semesterGet.js | 15 +++++++++++++++ sockets/get/userGet.js | 17 +++++++++++++++++ sockets/index.js | 3 ++- 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 sockets/get/agendaGet.js create mode 100644 sockets/get/groupGet.js create mode 100644 sockets/get/semesterGet.js create mode 100644 sockets/get/userGet.js diff --git a/sockets/get/agendaGet.js b/sockets/get/agendaGet.js new file mode 100644 index 0000000..c27a05c --- /dev/null +++ b/sockets/get/agendaGet.js @@ -0,0 +1,39 @@ +const models = require("../../models"); + +module.exports = socket => { + return async (data) => { + let startDate = data.startDate ? data.startDate : new Date(), endDate = data.endDate ? data.endDate : new Date(); + let options = {where: {startDate: startDate, endDate: endDate}, include: []}; + if (data.semesters) { + for (let semester of data.semesters) { + let s = await models.Semester.findByPk(semester); + if (!s) { + socket.emit("agendaGet", {error: {message: "semester_not_found"}}); + return + } else + options.include.push({model: models.Semester, where: {id: s.id}, require: true}); + } + } + if (data.groups) { + for (let group of data.groups) { + let g = await models.Group.findByPk(group); + if (!g) { + socket.emit("agendaGet", {error: {message: "group_not_found"}}); + return + } else + options.include.push({model: models.Group, where: {id: g.id}, require: true}); + } + } + if (data.teachers) { + for (let teacher of data.teachers) { + let t = await models.User.findByPk(teacher); + if (!t) { + socket.emit("agendaGet", {error: {message: "teacher_not_found"}}); + return + } else + options.include.push({model: models.User, where: {id: t.email}, require: true}); + } + } + socket.emit("agendaGet", await models.Event.findAll(options)); + } +}; diff --git a/sockets/get/groupGet.js b/sockets/get/groupGet.js new file mode 100644 index 0000000..12f16ba --- /dev/null +++ b/sockets/get/groupGet.js @@ -0,0 +1,21 @@ +const models = require("../../models"); + +module.exports = socket => { + return async (data) => { + let options = {where: {}, include: [{model: models.Semester, require: true}]}; + if (data.number) + options.where.number = data.number; + if (data.semester) { + let s = await models.Semester.findByPk(data.semester); + if (!s) { + socket.emit("groupGet", {error: {message: "semester_not_found"}}); + return + } + options.include[0].where = {id: s.id}; + } + if (data.users) + options.include.push({model: models.User, require: true}); + + socket.emit("groupGet", await models.Group.findAll(options)); + } +}; diff --git a/sockets/get/semesterGet.js b/sockets/get/semesterGet.js new file mode 100644 index 0000000..a7ef457 --- /dev/null +++ b/sockets/get/semesterGet.js @@ -0,0 +1,15 @@ +const models = require("../../models"); + +module.exports = socket => { + return async (data) => { + let options = {where: {}, include: []}; + if (data.name) + options.where.name = data.name; + if (data.year) + options.where.year = data.year; + if (data.groups) + options.include.push({model: models.Group, require: true}); + + socket.emit("semesterGet", await models.Semester.findAll(options)); + } +}; diff --git a/sockets/get/userGet.js b/sockets/get/userGet.js new file mode 100644 index 0000000..0b905c5 --- /dev/null +++ b/sockets/get/userGet.js @@ -0,0 +1,17 @@ +const models = require("../../models"); + +module.exports = socket => { + return async (data) => { + let options = {where: {}}; + if (data.email) + options.where.email = data.email; + if (data.firstName && data.lastName) { + options.where.firstName = data.firstName; + options.where.lastName = data.lastName; + } + if (data.permissions) + options.where.permissions = data.permissions; + + socket.emit("userGet", await models.User.findAll(options)); + } +}; diff --git a/sockets/index.js b/sockets/index.js index 604338a..b420000 100644 --- a/sockets/index.js +++ b/sockets/index.js @@ -10,6 +10,7 @@ module.exports = socket => { } else { socket.on("profileEdit", require("./profile/edit")(socket)); socket.on("logout", require("./logout")(socket)); + socket.on("agendaGet", require("./get/agendaGet")(socket)); } socket.emit("connected"); -} +}; From 2fec536d09907b5addd72e30a2f631eb8861a057 Mon Sep 17 00:00:00 2001 From: flifloo Date: Thu, 26 Nov 2020 12:28:55 +0100 Subject: [PATCH 4/4] Base of worker --- agenda/worker.js | 222 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 agenda/worker.js diff --git a/agenda/worker.js b/agenda/worker.js new file mode 100644 index 0000000..a26f450 --- /dev/null +++ b/agenda/worker.js @@ -0,0 +1,222 @@ +const https = require("https"); +const config = require("worker_threads").workerData; +const models = require("../models"); + +const reg_event = /(?:(?:BEGIN:VEVENT\nDTSTAMP:(?:[A-Z0-9]*?)\nDTSTART:([A-Z0-9]*?)\nDTEND:([A-Z0-9]*?)\nSUMMARY: {0,}([a-zéèàA-Z0-9-. \, \\/ô]*?)\nLOCATION:([a-zA-Zéèà0-9-. \,\\]*?)\nDESCRIPTION:(?:\\n){0,}(LP(?:[ a-zA-Z0-9]*))\\n((?:(?:[A-Z]*) (?:[A-Z]*)(?: (?:[A-Z]*)){0,}\\n){0,})(?:.*?)\nEND:VEVENT)|(?:BEGIN:VEVENT\nDTSTAMP:(?:[A-Z0-9]*?)\nDTSTART:([A-Z0-9]*?)\nDTEND:([A-Z0-9]*?)\nSUMMARY: {0,}((?:S(?:[A-Z0-9-]*)|M(?:[A-Z0-9-]*)(?:\/M(?:[A-Z0-9-]*)){0,}|Conférence)[a-zéèàA-Z0-9-. \, \\/]*?)\nLOCATION:([a-zA-Zéèà0-9-. \,\\]*?)\nDESCRIPTION:(?:\\n){0,}((?:(?:G[0-9]S[0-9]|S[0-9]|ASPE)\\n){0,})((?:(?:[A-Z]*) (?:[A-Z]*)(?: (?:[A-Z]*)){0,}\\n){0,})(?:.*?)\nEND:VEVENT))/gs; +const reg_location = /((?:[SH0-9][0-9]{2})|(?:(?:Préfa |Amphi)[0-9]))/g; +const reg_date = /([0-9]{4})([0-9]{2})([0-9]{2})T([0-9]{2})([0-9]{2})([0-9]{2})Z/; +const reg_classe = /(?:(LP)[ -]{0,}(.*)|(?:(G[0-9])(S[0-9])))/; +const base_url = config["edt"]; + +function fetchEvents(days = 1, TS_Start = new Date()) { + return new Promise((resolve, reject) => { + let start = new Date(TS_Start); + if (start == "Invalid Date") + start = new Date(); + let end = new Date(start); + end.setDate(end.getDate() + days); + + let url = base_url + start.getFullYear() + "-" + start.getMonth() + "-" + start.getDate() + "&lastDate=" + end.getFullYear() + "-" + end.getMonth() + "-" + end.getDate(); + https.get(url, (resp) => { + let data = ""; + resp.on("data", (chunk) => { + data += chunk; + }); + resp.on("end", () => { + let output = []; + data = data.replace(/\r/g, ""); + let m; + while ((m = reg_event.exec(data)) !== null) { + if (m.index === reg_event.lastIndex) { + reg_event.lastIndex++; + } + let event = []; + let promotion = []; + let classe = []; + /* + m = [ + FullMatch, + StartTime, + EndTime, + EventDescription ( SubjectID,Name,Class?) , + Location, + ClassGroup, + Teacher(s) + ] + */ + if (m[1] !== undefined) { + // LPXXXX + classe.push(m[5]); + } else { + m.splice(1, 6); + // GXSX | SX | ASPE + let csplit = m[5].split("\\n"); + csplit.pop(); + csplit.forEach(e => { + if (e === "ASPE") promotion.push(e); + if (/^S[0-9]$/.test(e)) promotion.push(e); + if (/^G[0-9]S[0-9]$/.test(e)) classe.push(e); + }); + } + event["title"] = m[3]; + event["promotion"] = promotion; + event["class"] = classe; + /* + Date + */ + let d1 = reg_date.exec(m[1]); + let d2 = reg_date.exec(m[2]); + event["startDate"] = new Date(d1[1] + "-" + d1[2] + "-" + d1[3] + "T" + d1[4] + ":" + d1[5] + ":" + d1[6] + ".00Z"); + event["endDate"] = new Date(d2[1] + "-" + d2[2] + "-" + d2[3] + "T" + d2[4] + ":" + d2[5] + ":" + d2[6] + ".00Z"); + /* + Location + */ + event["locations"] = []; + let loc; + while ((loc = reg_location.exec(m[4])) !== null) { + if (loc.index === reg_location.lastIndex) { + reg_location.lastIndex++; + } + event["locations"].push(loc[1]); + } + /* + Teachers + */ + event["teachers"] = m[6].split("\\n"); + event["teachers"].pop(); + output.push(event); + } + resolve(output); + }); + }).on("error", (err) => { + reject(err); + }); + }); +} + +function compare(a, b) { + if (a.length !== b.length) { + return false; + } + let set = {}; + a.forEach((i) => { + if (set[i] !== undefined) { + set[i]++; + } else { + set[i] = 1; + } + }); + let difference = b.every((i) => { + if (set[i] === undefined) { + return false; + } else { + set[i]--; + if (set[i] === 0) { + delete set[i]; + } + return true; + } + }); + return Object.keys(set) == 0 && difference; +} + +function compareGroups(list1, list2) { + return compare(list1, list2.map(g => [g.number, g.Semester.name, g.Semester.year])); +} + +function compareTeachers(list1, list2) { + return compare(list1, list2.map(t => t.lastName.toUpperCase() + " " + t.firstName.toUpperCase())); +} + +function compareSemesters(list1, list2) { + return compare(list1, list2.map(s => s.name)); +} + +async function updateDatabase() { + let events = await fetchEvents(365, new Date(2020, 9, 1)); + for (let event of await models.Event.findAll({ + include: [{ + model: models.Group, + include: {model: models.Semester, required: true} + }, models.User, models.Semester] + })) { + if (!events.find(e => (e.title === event.name && e.startDate.getTime() === event.startDate.getTime() && + e.endDate.getTime() === event.endDate.getTime() && e.locations.join(", ") === event.locations && + compareGroups(e.class, event.Groups) && compareTeachers(e.teachers, event.Users) && + compareSemesters(e.semesters, event.Semesters)))) + await event.destroy(); + else + delete events[events.indexOf(event)]; + } + + for (let event of events) { + let e = await models.Event.create({ + name: event.title, + startDate: event.startDate, + endDate: event.endDate, + locations: event.locations.join(", ") + }); + + let teachers = []; + for (let teacher of event.teachers) { + let t = await models.User.findOne({where: {permissions: 2, lastName: teacher[0], firstName: teacher[1]}}); + if (!t) + t = await models.User.create({ + email: teacher[1].toLowerCase().replace(" ", "-") + "." + teacher[0].toLowerCase().replace(" ", "-") + "@univ-lyon1.fr", + firstName: teacher[1], + lastName: teacher[0], + permissions: 2, + passwordHash: Math.round((Math.pow(36, 12 + 1) - Math.random() * Math.pow(36, 12))).toString(36).slice(1) + }); + teachers.push(t); + } + await e.addUsers(teachers); + + let semesters = []; + for (let semester of event.promotion) { + let s = await models.Semester.findOne({where: {name: semester, year: event.startDate.getFullYear()}}); + if (!s) + s = await models.Semester.create({ + name: semester, + year: event.startDate.getFullYear() + }); + semesters.push(s); + } + await e.addSemesters(semesters); + + let groups = []; + for (let group of event.class) { + let rGroup = reg_classe.exec(group); + let semesterName; + let groupName; + if (rGroup[1] !== undefined) { + // LP + semesterName = 'LP'; + groupName = rGroup[2]; + } else { + // GXSX + semesterName = rGroup[4]; + groupName = rGroup[3]; + } + + + let s = await models.Semester.findOne({where: {name: semesterName, year: event.startDate.getFullYear()}}); + if (!s) + s = await models.Semester.create({ + name: semesterName, + year: event.startDate.getFullYear() + }); + + let g = await models.Group.findOne({where: {number: groupName, SemesterId: s.id}}); + if (!g) + g = await models.Group.create({ + number: groupName, + SemesterId: s.id + }); + groups.push(g); + } + await e.addGroups(groups); + } +} + +updateDatabase().then(() => setInterval(updateDatabase, 30000));