Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

load assignment psets #9

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

load assignment psets #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"commander": "^10.0.0",
"filenamify": "^5.1.1",
"fs-extra": "^11.1.1",
"jsdom": "20.0.0",
"lodash": "^4.17.21",
"superagent": "^8.0.9"
},
Expand Down
43 changes: 42 additions & 1 deletion src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ import path from 'path';
import { program } from 'commander';
import filenamify from "filenamify";
import lodash from 'lodash';
import jsdom from 'jsdom';


const { JSDOM } = jsdom;

program
.option('-c --course [course]', "Course to download (name or code)", String)
Expand Down Expand Up @@ -197,6 +198,45 @@ async function downloadModules(c: Course, courseDir: string) {
}


async function downloadAssignments(c: Course, courseDir: string) {
// load files stored in assignment bodies
// files are flat, folders data needed to replicate canvas folder structure
const assignments = await getJson(`${options.url}/courses/${c.id}/assignments`);
const folder = path.resolve(courseDir, "psets/");
await fs.mkdirs(folder)
for (let assign of assignments){
// match <a> link name and url from html desc using regex
let links = (new JSDOM("<!DOCTYPE html>" + assign.description)).window.document.querySelectorAll("a");
links.forEach(async (link) => {
let href = link.getAttribute("data-api-endpoint");
if (!href) return

// this can be any of a number of different interfaces.
// it might contain a download link for a file which is not available under the /files api
const thing = await getJson(href)

if (!thing.url) return

const destPath = path.resolve(folder, link.textContent);


await fIfNeeded(
()=>new Promise((resolve, reject) => {
var stream = fs.createWriteStream(destPath);
stream.on('finish', resolve);
stream.on('error', reject)
request.get(thing.url)
.set('Authorization', `Bearer ${options.token}`)
.pipe(stream);
}),
destPath,
new Date(thing.modified_at)
)
})
}
}


async function downloadPages(c: Course, courseDir: string) {
const pages = await getJson(`${options.url}/courses/${c.id}/pages`) as Page[];
if (!pages) return;
Expand Down Expand Up @@ -252,6 +292,7 @@ async function run() {
const courseDir = path.resolve(targetFolder, santizeFilename(c.name)+"_"+c.id);
await fs.mkdirp(courseDir);

await downloadAssignments(c, courseDir).catch(console.error);
await downloadFiles(c, courseDir).catch(console.error);
await downloadPages(c, courseDir).catch(console.error);
await downloadAnnouncements(c, courseDir).catch(console.error);
Expand Down