diff --git a/.gitignore b/.gitignore index a37fc58..bbb2170 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .git node_modules/ node_modules/ -.vscode \ No newline at end of file +data/ +.vscode -- 2.51.2 From fbf67463e6e29f3a38280baedc10a5aa327e0e49 Mon Sep 17 00:00:00 2001 From: Chris Vanderloo Date: Mon, 18 Feb 2019 12:00:44 -0500 Subject: [PATCH 2/3] Adds cron job for auto-grabbing YACS data --- index.js | 13 ++++++++----- package-lock.json | 19 +++++++++++++++++++ package.json | 3 ++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index e8f1d20..c8fcf48 100644 --- a/index.js +++ b/index.js @@ -1,15 +1,13 @@ -const request = require("request"); const express = require("express"); -const cookieParser = require("cookie-parser"); const path = require('path'); const api = require('./api.js'); +const cron = require('node-cron'); +const runUpdate = require('./update_data.js'); var app = express(); -app.use(cookieParser()); - // YACS -require('./routes/yacs.js')(app, api.yacs); +require('./routes/yacs.js')(app, api.yacs, runUpdate); // SIS require('./routes/sis.js')(app, api.sis); @@ -24,4 +22,9 @@ app.get("/verify_status", (req, res) => { app.listen(8080, () => { console.log("Listening on port 8080"); +}); + +cron.schedule('0 0 * * *', () => { + console.log("Updating YACS data"); + runUpdate(); }); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index ffd1eb0..e165a27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2132,6 +2132,15 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, + "node-cron": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-2.0.3.tgz", + "integrity": "sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg==", + "requires": { + "opencollective-postinstall": "^2.0.0", + "tz-offset": "0.0.1" + } + }, "nodemon": { "version": "1.18.9", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.18.9.tgz", @@ -2261,6 +2270,11 @@ "wrappy": "1" } }, + "opencollective-postinstall": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz", + "integrity": "sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw==" + }, "p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -3008,6 +3022,11 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, + "tz-offset": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/tz-offset/-/tz-offset-0.0.1.tgz", + "integrity": "sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==" + }, "undefsafe": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.2.tgz", diff --git a/package.json b/package.json index 30ec540..0158db5 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "nodemon ." + "start": "nodemon --ignore 'data/**/*' ." }, "author": "", "license": "ISC", @@ -13,6 +13,7 @@ "cheerio": "^1.0.0-rc.2", "cookie-parser": "^1.4.3", "express": "^4.16.4", + "node-cron": "^2.0.3", "nodemon": "^1.18.9", "puppeteer": "^1.11.0", "request": "^2.88.0", -- 2.51.2 From fe550156614b7afcf1aa50fa8cf78f01c83bad34 Mon Sep 17 00:00:00 2001 From: Chris Vanderloo Date: Mon, 18 Feb 2019 12:01:01 -0500 Subject: [PATCH 3/3] YACS data dump to server --- api.js | 6 ++++++ routes/yacs.js | 13 +++++-------- sources.json | 3 ++- update_data.js | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 update_data.js diff --git a/api.js b/api.js index 2b3261a..55b7bbd 100644 --- a/api.js +++ b/api.js @@ -76,6 +76,12 @@ module.exports = { if (!err) next(null, body); else next(err); }); + }, + courses: (dep_id, next) => { + request.get(sources.yacs.courses + dep_id + "&show_periods=true&show_sections=true&", (err, res, body) => { + if (!err) next(null, body); + else next(err); + }); } } diff --git a/routes/yacs.js b/routes/yacs.js index a882e1e..605a833 100644 --- a/routes/yacs.js +++ b/routes/yacs.js @@ -1,11 +1,8 @@ -module.exports = (app, api) => { +module.exports = (app, api, update) => { + app.get("/api/update", (req,res) => { + update(); + res.send("Ran update"); + }); - app.get("/api/departments", (req,res) => - api.departments((err, body) => { - if (!err) res.send(body); - else res.send(err); - })); - - } \ No newline at end of file diff --git a/sources.json b/sources.json index ad53e1d..0ab298f 100644 --- a/sources.json +++ b/sources.json @@ -6,6 +6,7 @@ "get_student_menu": "https://sis.rpi.edu/rss/twbkwbis.P_GenMenu?name=bmenu.P_StuMainMnu" }, "yacs": { - "departments": "https://yacs.cs.rpi.edu/api/v5/schools.json?show_departments=true" + "departments": "https://yacs.cs.rpi.edu/api/v5/schools.json?show_departments=true", + "courses": "https://yacs.cs.rpi.edu/api/v5/courses.json?department_id=" } } \ No newline at end of file diff --git a/update_data.js b/update_data.js new file mode 100644 index 0000000..404668c --- /dev/null +++ b/update_data.js @@ -0,0 +1,38 @@ +const api = require('./api.js').yacs; +const fs = require('fs'); + +var num_files = 0; +var total = 0; + +function writeDepartment (dep) { + api.courses(dep.id, (err, body) => { + if (err) console.log(err); + else + fs.writeFile("./data/courses/"+dep.id+".json", body, (err) => { + if (err) return console.log(err); + num_files++; + console.log("\x1b[1A\x1b[0K- ["+num_files+"/"+total+"] Saved '"+dep.id+".json'"); + }); + }) +} + +module.exports = () => { + num_files = 0; + api.departments((err, body) => { + if (err) console.log(err); + else + fs.writeFile("./data/departments.json", body, (err2) => { + if (err2) + return console.log(err2); + console.log("- Saved 'departments.json'"); + var obj = JSON.parse(body) + total = 0; + obj.schools.forEach((school) => total+=school.departments.length); + obj.schools.forEach( + (school) => school.departments.forEach((item) => { + writeDepartment(item) + }) + ); + }); + }); +} \ No newline at end of file