diff options
| -rw-r--r-- | Cont/index.html | 2 | ||||
| -rw-r--r-- | Functions/dateFuncs.js | 21 | ||||
| -rw-r--r-- | Functions/open.js | 23 | ||||
| -rw-r--r-- | README.md | 18 | ||||
| -rw-r--r-- | dbparse.js | 192 | ||||
| -rw-r--r-- | food.js | 61 | ||||
| -rw-r--r-- | parseClasses.js | 7 | ||||
| -rw-r--r-- | scrape.js | 75 | ||||
| -rw-r--r-- | server.js | 68 | ||||
| -rw-r--r-- | update.js | 11 |
10 files changed, 229 insertions, 249 deletions
diff --git a/Cont/index.html b/Cont/index.html index 95dcb32..a9d83e7 100644 --- a/Cont/index.html +++ b/Cont/index.html @@ -50,6 +50,8 @@ <div class="float-block"> <h2 class="shadow">\(food-header\)</h2> <p>\(food\)</p> + <h2 class="shadow">\(vege-header\)</h2> + <p>\(vege\)</p> </div> </div> </main> diff --git a/Functions/dateFuncs.js b/Functions/dateFuncs.js index 10c4250..9b1237a 100644 --- a/Functions/dateFuncs.js +++ b/Functions/dateFuncs.js @@ -17,7 +17,26 @@ function approxDate(d) return new Date(`${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, "0")}-${d.getDate().toString().padStart(2, "0")}`); } +function weekdayToNumber(s) +{ + const weekdays = [ + /ma.*/i, + /ti.*/i, + /ke.*/i, + /to.*/i, + /pe.*/i, + /la.*/i, + /su.*/i + ]; + for(let day = 0; day < weekdays.length; day++) + { + if (s.match(weekdays[day])) + return day; + } +} + module.exports = { fromString: stringToDate, - between: isBetweenDates + between: isBetweenDates, + weekdayToNumber: weekdayToNumber }
\ No newline at end of file diff --git a/Functions/open.js b/Functions/open.js index f04201b..f532721 100644 --- a/Functions/open.js +++ b/Functions/open.js @@ -1,4 +1,5 @@ const fs = require("fs"); +const https = require("https"); function openFile(path) { @@ -13,4 +14,24 @@ function openFile(path) }); } -exports.file = openFile; +async function urlOpen(path) +{ + return new Promise((resolve, reject) => + { + let req = https.get(path, res => + { + res.on("data", resolve); + }); + }); + req.on("error", e => + { + console.error(e); + }); + req.end(); +} + + +module.exports = { + file: openFile, + url: urlOpen +};
\ No newline at end of file @@ -68,6 +68,14 @@ CREATE TABLE exams ( message VARCHAR(256), PRIMARY KEY (start, end) ); +CREATE TABLE foods ( + week INT, + day INT, + vegetarian TINYINT, + header VARCHAR(15), + dateString VARCHAR(13), + food VARCHAR(256) +); ``` > Note that if you had some information in a former database that you don't update manually, it will be lost. @@ -103,20 +111,20 @@ Lets assume the following filesystem that contains also all of the server code: ``` shifts.txt Classes -| oldclasses.txt -| newclasses.txt +| classes.txt ``` -Where shifts.txt contains the shifts, `oldclasses.txt` contains the classes of the old curriculum and `newclasses` the classes of the new learning curriculum. +Where shifts.txt contains the shifts and `classes.txt` contains the classes. -You can get the shifts from junu's food shift message through Wilma. The classes should be tab delimited text files. You can get them easily by copy-pasting them from the eg. LibreOffice from "Kurssitarjottimet". Provide only the classes of one period, not all of them. +You can get the shifts from junu's food shift message through Wilma. The `classes` should be a tab delimited text file. You can get it easily by copy-pasting its contents from eg. LibreOffice from "Kurssitarjottimet". Provide only the classes of one period, not all of them. You can give several class files, if needed. Then just run the following code in node.js: ``` const updateDB = require("./update.js"); const openFile = require("./Functions/open.js").file; const dbcredentials = await openFile("../dblogin.txt"); -await updateDB.update("./shifts.txt", ["./Classes/oldclasses.txt", "./Classes/newclasses.txt"], dbcredentials); +await updateDB.update(dbcredentials, "./shifts.txt", "./Classes/classes.txt"); ``` +If you have several files with classes, just append them to the parameters of `updateDB.update`. ## Notifying of unusual food shifts (eg. during exams) Currently the notifications have to be added manually to the MySQL database. Here's an example: @@ -1,6 +1,5 @@ +const weekdayToNumber = require("./Functions/dateFuncs.js").weekdayToNumber; - -// String searching function getCharAmount(s, c) { let n = 0; @@ -57,7 +56,7 @@ function findExpression(data, expr, start = 0) return start; } -// Normalizing + function parseCluttered(s) { if (!(typeof s === "string")) @@ -65,145 +64,85 @@ function parseCluttered(s) return s.replaceAll(".", "").replaceAll(" ", "").toUpperCase(); } -// Class parsing -async function writeClasses(classData, DB) -{ - classData = parseCluttered(classData) + "\n"; // newline so that loop can find last value - await DB.query_raw("DELETE FROM classes"); - // parse data to dict - let i = 0; - while (i < classData.length) - { - let separator = getNextChar(classData, ":", i); - if (separator === -1) - break; - let lineEnd = getNextChar(classData, "\n", i); - let key = classData.substring(i, separator); - let val = classData.substring(separator + 1, lineEnd); - i = lineEnd + 1; - let res = await DB.execute("INSERT INTO classes VALUES (?, ?)", [key, val]); - } -} - -async function parseLine(data, day, shift, DB) +async function writeShifts(data, DB) { - // "preprocessing" - let i = 0; - let courses = []; - let teachers = []; - const toRemove = " ja KAHDEN TUTKINNON OPINNOT 1., 2. ja 3. VUOSITASON RYHMÄT "; - if (data.substring(data.length - toRemove.length, data.length) === toRemove) - data = data.substring(0, data.length - toRemove.length); - data = data.replaceAll(",", "").replaceAll("ja ", "").replaceAll(" + ", "+"); + let deletions = await Promise.all([ + DB.query_raw("DELETE FROM shifts"), + DB.query_raw("DELETE FROM shiftnames") + ]); - while (i < data.length) - { - if (data[i] === "+") + const dbOperations = []; + const shiftRegex = /((?:MAANANTAI|TIISTAI|KESKIVIIKKO|TORSTAI|PERJANTAI)?.*)\s*(RUOKAILU.*)\s*(.*)/gmi; + const shifts = data.matchAll(shiftRegex); + let weekday; + let shiftId = 1; + for(const shift of shifts) + { + if (shift[1] !== "") { - - nextSpace = getNextChar(data, " ", i); - let nextNextSpace = getNextChar(data, " ", nextSpace + 1); - if (nextNextSpace === -1) - nextNextSpace = data.length; - data = `${data.substring(0, i)} ${data.substring(nextSpace + 1, nextNextSpace)} ${data.substring(i + 1, data.length)}`; - i = nextNextSpace - 1; + weekday = weekdayToNumber(shift[1]); + shiftId = 1; } - i++; - } - nextSpace = 0; - i = 0; - const getElement = list => - { - nextSpace = getNextChar(data, " ", i); - if (nextSpace === -1) - nextSpace = data.length; - list.push(data.substring(i, nextSpace)); - i = nextSpace + 1; - } + dbOperations.push( + writeShift(weekday, shiftId, shift[2], shift[3], DB) + ); - do - { - getElement(courses); - getElement(teachers); - } while (i < data.length) - - let values = "VALUES"; - for(let el = 0; el < courses.length; el++) - { - values += ` ROW(${day}, ${shift}, '${courses[el]}', '${teachers[el]}', NULL),`; + shiftId++; } - values = values.substring(0, values.length - 1); - return DB.execute(`INSERT INTO shifts ${values}`, []); -} -async function parseDay(data, day, DB) -{ - let i = getToLineStartingWith(data, "RUOKAILUVUORO"); - let indexOfShift = 1; - while (i !== -1) - { - let endOfLine = getNextChar(data, "\n", i); - // Insert the food shift name - let shiftName = DB.execute("INSERT INTO shiftnames VALUES (?, ?, ?)", [day, indexOfShift, data.substring(i, endOfLine)]); - // get to the teachers & courses - i = endOfLine + 1; - i = getNextChar(data, "\n", i) + 1; - if (getNextChar(data, "\n", i) === -1) - endOfLine = data.length; - else - endOfLine = getNextChar(data, "\n", i); - let unparsedIndexes = data.substring(i, endOfLine); - - // do the magic - let lineParse = parseLine(unparsedIndexes, day, indexOfShift, DB); - - i = getToLineStartingWith(data, "RUOKAILUVUORO", i); - indexOfShift++; - await Promise.all([shiftName, lineParse]); - } + await dbOperations; return 0; } -async function writeShifts(data, DB) +async function writeShift(weekday, shiftId, shiftLine, courseLine, DB) { - weekdays = ["MAANANTAI", "TIISTAI", "KESKIVIIKKO", "TORSTAI", "PERJANTAI"]; - let deletions = Promise.all([ - DB.query_raw("DELETE FROM shifts"), - DB.query_raw("DELETE FROM shiftnames") - ]); - - // iterate over the weekdays - let i = 0; - for (let day = 0; day < weekdays.length; day++) + const dbOperations = []; + // Shift names + dbOperations.push( + DB.execute( + "INSERT INTO shiftnames VALUE (?, ?, ?)", + [weekday, shiftId, shiftLine] + ) + ); + + // Shift contents + const courseRegex = /(?:[a-ö]{2,3}\d{2,3} ?\+? ?)+ [a-ö]{4}/gi; + const courses = courseLine.matchAll(courseRegex); + for(const course of courses) { - // find the start of the shifts of the day - i = getNextChar(data, "\n", findExpression(data, weekdays[day], i)) + 1; - - // find the end of the shifts of the day - let end = [ - data.length, - findExpression(data, weekdays[day + 1], i) - ][+(day !== weekdays.length - 1)]; + const _lastSpace = course[0].lastIndexOf(" "); + const courseNames = course[0].substring(0, _lastSpace).split(/ ?\+ ?/); + const teacherName = course[0].substring(_lastSpace + 1); - await deletions; // wait for deletion to get completed before proceeding to write new data to the table - - // do the magic: - let shifts = data.substring(i, end); - await parseDay(shifts, day, DB); - - i = end; + // For loop is needed, because some courses are marked like KE16+KE26 MATI + async function handleCourse(courseName, teacherName) + { + let className = await DB.execute( + "SELECT class FROM classes WHERE course=?", + [courseName] + ); + className = className[0]; + if (className !== undefined) + className = className.class; + else + className = null; + + dbOperations.push(DB.execute( + `INSERT INTO shifts VALUES (${weekday}, ${shiftId}, ?, ?, ?)`, + [courseName, teacherName, className] + )); + } + for(const courseName of courseNames) + { + dbOperations.push(handleCourse(courseName, teacherName)); + } } - const courses = await DB.query_raw("SELECT * FROM classes"); - const results = []; - for (let course = 0; course < courses.length; course++) - { - results.push(DB.query("UPDATE shifts SET class = ? WHERE course = ?", [courses[course].class, courses[course].course])); - } - await Promise.all(results); + await Promise.all(dbOperations); return 0; } + async function getShift(day, index, DB) { @@ -252,9 +191,8 @@ async function getRandomIndex(day, DB) exports.indexType = getIndexType; exports.cluttered = parseCluttered; -exports.find = findExpression; -exports.getNextChar = getNextChar; -exports.classes = writeClasses; exports.build = writeShifts; exports.get = getShift; exports.randomIndex = getRandomIndex; +exports.find = findExpression; +exports.getNextChar = getNextChar;
\ No newline at end of file @@ -0,0 +1,61 @@ +const parse = require("./dbparse.js"); +const open = require("./Functions/open.js"); +const { weekdayToNumber } = require("./Functions/dateFuncs.js"); + +function* scrapeFood(data) +{ + const foodRegex = /<title>(\w{2} (?:\d\d?\.){2}\d{4})<\/title><description><!\[CDATA\[(Lounas) ?:? ?(.*?)(Kasvislounas) ?:? ?(.*?)]]><\/description>/gm; + const foods = data.matchAll(foodRegex); + for(const food of foods) + { + yield [ + weekdayToNumber(food[1]), // index + food[1], // header (date) + [food[2], food[3]], // first header, first food + [food[4], food[5]] // second header, second food + ]; + } +} + +async function buildFoods(DB) +{ + await DB.query_raw("DELETE FROM foods"); + let foodData = await Promise.all([ + open.url(getFoodLink(1)), + open.url(getFoodLink(2)) + ]); + foodData = foodData.map((f) => f.toString("utf-8")); + const foodInitOperations = []; + const foods = foodData.map(f => scrapeFood(f)); + for(let week = 1; week <= 2; week++) + { + for(const food of foods[week - 1]) + { + foodInitOperations.push(DB.execute("INSERT INTO foods VALUES (?, ?, FALSE, ?, ?, ?)", [ + week, + food[0], + food[2][0], + food[1], + food[2][1] + ])); + foodInitOperations.push(DB.execute("INSERT INTO foods VALUES (?, ?, TRUE, ?, ?, ?)", [ + week, + food[0], + food[3][0], + food[1], + food[3][1] + ])); + } + } + await Promise.all(foodInitOperations); +} + +function getFoodLink(week) +{ + return `https://eruokalista.lohja.fi/AromieMenus/FI/Default/Lohja/Koulut/Rss.aspx?Id=97f76449-f57c-4217-aede-b5f9dbf2b41e&DateMode=${week}`; +} + + +exports.foods = scrapeFood; +exports.link = getFoodLink; +exports.build = buildFoods;
\ No newline at end of file diff --git a/parseClasses.js b/parseClasses.js index ffd8e7f..b114f33 100644 --- a/parseClasses.js +++ b/parseClasses.js @@ -56,13 +56,12 @@ function addToDBFromLists(DB, l1, l2, l1cond, l2cond) } } -async function parseClasses(path1, path2, DB) +async function parseClasses(DB, ...paths) { let parsed = []; await DB.query_raw("DELETE FROM classes"); - parsed.push(parseClassData(path1, DB)); - if (path2 !== undefined) - parsed.push(parseClassData(path2, DB)); + for(const path of paths) + parsed.push(parseClassData(path, DB)) return await Promise.all(parsed); } diff --git a/scrape.js b/scrape.js deleted file mode 100644 index c7a5743..0000000 --- a/scrape.js +++ /dev/null @@ -1,75 +0,0 @@ -const https = require("https"); -const parse = require("./dbparse.js"); -const fs = require("fs"); -const events = require("events"); - -async function urlOpen(path) -{ - return new Promise((resolve, reject) => - { - let req = https.get(path, res => - { - res.on("data", resolve); - }); - }); - req.on("error", e => - { - console.error(e); - }); - req.end(); -} - -async function scrapeFood(url) -{ - let data = await urlOpen(url); - data = data.toString("utf-8"); - - let foodList = []; - const weekdays = ["ma", "ti", "ke", "to", "pe", "la", "su"]; - - let titleTags = ["<title>", "</title>"]; - let foodTags = ["<![CDATA[", "]]>"]; - const getSpan = (data, tags, i = 0) => - { - return [ - parse.find(data, tags[0], i) + tags[0].length, - parse.find(data, tags[1], i) - ]; - } - let mainTitle = parse.find(data, titleTags[1]) + titleTags[1].length; - let titleSpan = getSpan(data, titleTags, mainTitle); - let foodSpan = getSpan(data, foodTags); - - while ( - (titleSpan[0] !== -1) - && (titleSpan[1] !== -1) - && (foodSpan[0] !== -1) - && (foodSpan[1] !== -1) - ) - { - let title = data.substring(titleSpan[0], titleSpan[1]); - let food = data.substring(foodSpan[0], foodSpan[1]); - - let weekdayIndex = weekdays.findIndex(val => { return val === title.substring(0, 2); }); - if (weekdayIndex !== -1) - foodList[weekdayIndex] = [title, neatify(food)]; - - titleSpan = getSpan(data, titleTags, foodSpan[1]); - foodSpan = getSpan(data, foodTags, titleSpan[1]); - } - - return foodList; -} - -function getFoodLink(week) -{ - return `https://eruokalista.lohja.fi/AromieMenus/FI/Default/Lohja/Koulut/Rss.aspx?Id=97f76449-f57c-4217-aede-b5f9dbf2b41e&DateMode=${week}`; -} - -function neatify(food) -{ - return food.replaceAll(")", ")<br>").replaceAll(" :", ":").replaceAll(":", ":<br>"); -} - -exports.food = scrapeFood; -exports.link = getFoodLink; @@ -1,10 +1,10 @@ //const http = require("http"); const https = require("https"); const url = require("url"); -const scrape = require("./scrape.js"); +const food = require("./food.js"); const SQL_DBS = require("./database.js"); const DBPARSE = require("./dbparse.js"); -const openFile = require("./Functions/open.js").file; +const open = require("./Functions/open.js"); const strFuncs = require("./Functions/stringFuncs.js"); const dateFuncs = require("./Functions/dateFuncs.js"); @@ -27,14 +27,13 @@ async function init() let visitorCount = 0; // await for needed things in async - let [foodsThisWeek, foodsNextWeek, dbcredentials, httpsKey, httpsCert] = await Promise.all([ - scrape.food(scrape.link(1)), - scrape.food(scrape.link(2)), - openFile("../dblogin.txt"), - openFile("../Certificate/key.pem"), - openFile("../Certificate/cert.pem") + let [dbcredentials, httpsKey, httpsCert] = await Promise.all([ + open.file("../dblogin.txt"), + open.file("../Certificate/key.pem"), + open.file("../Certificate/cert.pem") ]); - + + // https options, you need to get a certificate in the file ../Certificate for the server to work const httpsOpts = { key: httpsKey, @@ -44,8 +43,15 @@ async function init() // get the MySQL DB connection const SQLDB = new SQL_DBS.Database(JSON.parse(dbcredentials)); - // get the food "database" - const foods = [foodsThisWeek, foodsNextWeek]; + // Add the foods to the database + await food.build(SQLDB); + setInterval( + () => + { + food.build(SQLDB); + }, + 7 * 24 * 60 * 60 * 1000 + ); // server code async function server(req, res) @@ -79,7 +85,6 @@ async function init() "path": path, "path404": errorPath, "query": q.query, - "foods": foods, "sqldb": SQLDB }; @@ -157,10 +162,9 @@ async function buildMain(args) // get the passed arguments const path = args["path"]; const query = args["query"]; - const foods = args["foods"]; const index = query.index; const SQLDB = args["sqldb"]; - const data = await openFile(path); + const data = await open.file(path); let data_string = data.toString("utf-8"); // here are the things to replace in the html page @@ -242,19 +246,21 @@ async function buildMain(args) data_string = data_string.replace('<div id="shift-result" class="float-block">', '<div id="shift-result" class="float-block" style="display: none;">'); // get the food - let food; - food = foods[ +(day < actualDay) ][day]; - if (food !== undefined) - { - res["food-header"] = food[0]; - res["food"] = food[1]; - } - else - { - res["food-header"] = weekdays[day]; - res["food"] = "Päivälle ei löytynyt ruokaa"; - } - res["food-header"] = `Kouluruoka ${res["food-header"]}:`; + const week = +(day < actualDay) + 1; // Week = 1 if day is not past + const [food, vege] = await Promise.all([ + SQLDB.execute( + "SELECT header, datestring, food FROM foods WHERE week=? AND day=? AND vegetarian=FALSE", + [week, day] + ), + SQLDB.execute( + "SELECT header, datestring, food FROM foods WHERE week=? AND day=? AND vegetarian=TRUE", + [week, day] + ) + ]); + res["food-header"] = `${food[0].header} ${food[0].datestring}`; + res["vege-header"] = vege[0].header; + res["food"] = food[0].food; + res["vege"] = vege[0].food; data_string = build_replace(data_string, res); @@ -264,7 +270,7 @@ async function buildMain(args) async function buildDevs(args) { const path = args["path"]; - const data = await openFile(path); + const data = await open.file(path); const DB = args["sqldb"]; let res = ""; @@ -286,7 +292,7 @@ async function buildDevs(args) async function build404(args) { args["path"] = args["path"].substring("./Cont".length); - const data = await openFile(args["path404"]); + const data = await open.file(args["path404"]); const data_string = data.toString("utf-8"); return data_string.replace("\\(path\\)", args["path"]); } @@ -294,14 +300,14 @@ async function build404(args) async function buildDefault(args) { const path = args["path"]; - const data = await openFile(path); + const data = await open.file(path); return data.toString("utf-8"); } async function buildImage(args) { const path = args["path"]; - const data = await openFile(path); + const data = await open.file(path); return data; } @@ -4,13 +4,14 @@ const parseClasses = require("./parseClasses.js").classes; const parse = require("./dbparse.js"); // Run this if you want to build the database from text files -async function buildDB(shiftfile = "./shifts.txt", classfile = "./classes.txt", dbcredentials) +async function buildDB(dbcredentials, shiftPath, ...classfiles) { - let shiftCont = await openFile(shiftfile); + let shiftCont = await openFile(shiftPath); const DB = new database.Database(JSON.parse(dbcredentials)); shiftCont = shiftCont.toString("utf-8").replaceAll("\r", ""); // \r because of the \r\n newline on windows which creates problems + await Promise.all([ - parseClasses(classfile[0], classfile[1], DB), + parseClasses(DB, ...classfiles), parse.build(shiftCont, DB) ]); return 0; @@ -20,6 +21,6 @@ exports.update = buildDB; // Example call: /* const openFile = require("./Functions/open.js").file; -const dbcredentials = openFile("../dblogin.txt"); -await updateDB.update("./shifts.txt", ["./Kurssitarjottimet/2016Classes.txt", "./Kurssitarjottimet/NewClasses.txt"], dbcredentials); +const dbcredentials = await openFile("../dblogin.txt"); +await updateDB.update(dbcredentials, "./shifts.txt", "./Kurssitarjottimet/2016Classes.txt", "./Kurssitarjottimet/NewClasses.txt"); */
\ No newline at end of file |