Rewrite in node

.gitignore

@@ -0,0 +1 @@
+fcc_progress.json

README.md

@@ -1,35 +1,27 @@
+# Greenville Codes Student Progress Tracker
+
 ## Overview
 
-This script takes a newline-delimited list of FreeCodeCamp profile URLs and returns a report on their progress as a newline-delimited list of tab-delimited percentages. 
+This script takes a newline-delimited list of FreeCodeCamp profile URLs and returns a report on their progress as a newline-delimited list of comma-separated percentages. These can be easily pasted into a Google sheet used for tracking a cohort's progress through the Greenville Codes program.
 
 ## Installation
 
-1. [Install Tampermonkey.](https://chrome.google.com/webstore/detail/tampermonkey/dhdgffkkebhmkfjojejmpbldmpobfkfo)
-2. Navigate to the Utilities tab in Tampermonkey options.
-3. Paste the following address into the input marked "URL" and click "Import" <https://raw.githubusercontent.com/dacohenii/fcc-progress/master/bulk-fcc-progress.js> 
-4. Confirm the installation.
-
-![Installation](assets/install.gif)
+Requires Node v7.6 or higher (testing done with v10.1.0).
+Clone the repo and cd into it.
+Install the dependencies with `npm install`.
 
 ## Usage
 
-_Note: This script sends a request for all these URLs essentially at the same time, so and hasn't been tested for more than ~30 at a time. Too many more and their serers might not appreciate it._
-
-1. Copy a spreadsheet column (or newline-separated list) of URLs to be scraped to the clipboard.
-    - empty cells / lines are okay and will be preserved on paste.
-2. Invoke the script via the Tampermonkey menu.
-3. A prompt should come up. Paste the addresses into the prompt and submit it.
-4. At this point, it will start retrieving results.
-5. A notification should appear to let you know the process is complete. When this happens select the top-left destination cell and paste the result. 
-
-## Misc. Notes
+Create the FreeCodeCamp curriculum map with `node map_fcc_path.js`.
+It will create fcc_path.json or overwrite the existing file.
 
-The main functionality is in `bulk-fcc-progress.js`. 
+Use `node get_fcc_progress.js` to get a detailed progress report for a single FCC profile. You can specify to track the overall FCC curriculum or the Greenville Codes specific path in the file head.
 
-`fcc-progress.js` scrapes a single profile and must be run directly from that page. 
+Use `node get_bulk_fcc_progress.js` to get a csv report of a full list of student profiles. By default, these are pulled from fcc_profiles.txt, but this can be modified in the file head.
 
-This was developed and tested using Tampermonkey and Chrome 60. YMMV. 
+## Troubleshooting
+Running `node get_bulk_fcc_progress.js` with an invalid url will return an error. This includes newline characters at the end of the file! I'll work on fixing this.
 
-The lessons are currently hard-coded in the files. This can be pulled dynamically in the future, but they don't seem to change too often.
+I ran into a few problems getting this to work on another computer. My errors were related to dependencies for Chrome. I resolved them by following the advice [here](https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md).
 
-Also, it's a web scraper, which is likely to break at some point when they change their website, but that's okay, becuase the next version will probably use an API, 
+`npm install` loads the dependencies locally, so `ldd chrome | grep not` won't work in the base directory. You'll have to navigate all the way down to the node_modules directory for puppeteer, through the .local-chromium directory until you find the chrome binary, then run the command. This should give you the missing dependencies that need to be installed.

fcc_profiles.txt

@@ -0,0 +1,3 @@
+https://www.freecodecamp.org/magoun
+https://www.freecodecamp.org/thirtythreeb
+https://www.freecodecamp.org/jaaron0606

get_bulk_fcc_progress.js

@@ -0,0 +1,131 @@
+/**
+ * Scrapes multiple user profile from fcc_profiles.txt and 
+ * compares completed challenges against the Greenville Codes curriculum. 
+ * 
+ * Track Greenville Codes progress with gvl_codes_path.json.
+ * 
+ * Usage: node bulk_fcc_progress.js
+ * 
+ */
+
+const outputFile = 'student_progress.csv',
+      urlFile = 'fcc_profiles.txt',
+      trackFile = 'gvl_codes_path.json';
+
+const fs = require('fs');
+const puppeteer = require('puppeteer');
+const URL = require('url');
+
+// Function for filtering bad inputs from urlFile
+let validateURL = (str) => {
+  const url = URL.parse(str);
+
+  if (url.hostname) {
+    return true;
+  } else {
+    console.log('Skipping ' + str + ' (invalid URL)');
+    return false;
+  }
+};
+
+// Read list of profile urls
+// Returns an array of profile URLs
+let readURLs = (file) => {
+  let profiles = fs.readFileSync(file, 'utf8');
+  console.log('Reading profile urls from ' + urlFile);
+
+  // Splitting the return on new line yields an array
+  return profiles.split('\n').filter(validateURL);
+};
+
+// Fetch html using puppeteer
+let getHTML = async (url) => {
+  console.log('Getting HTML from ' + url);
+
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+
+  // Wait for React components to load
+  await page.goto(url, { waitUntil: 'networkidle0'});
+
+  const html = await page.content();
+  await browser.close();
+
+  return html;
+};
+
+// Parse the HTML for completed challenges
+// Returns an array of challenge names
+let parseHTML = (html) => {
+  const $ = require('cheerio').load(html);
+  let challenges = [];
+
+  $('tr', 'tbody').each(function (index, element) {
+    let challenge = $(this).find('a').text();
+    challenges.push(challenge);
+  });
+
+  return challenges;
+};
+
+// Compare progress to overall curriculum
+// Returns a comma delimited string
+let getProgress = (completedChallenges) => {
+  const rawJSON = fs.readFileSync(trackFile);
+  let challengeMap = JSON.parse(rawJSON);
+
+  // Inconsistent capitalization, thus map to lower case
+  completedChallenges = completedChallenges.map(x => x.toLowerCase());
+
+  let progress = '';
+
+  // Loop through curriculum JSON
+  for (let section of Object.keys(challengeMap)) {
+    for (let subsection of Object.keys(challengeMap[section])) {
+      let numChallenges = challengeMap[section][subsection].length;
+
+      // Inconsistent capitalization, thus map to lower case
+      let subsectionArray = challengeMap[section][subsection].map(x => x.toLowerCase());
+
+      let numCompleted = subsectionArray.reduce((acc, val) => {
+        return completedChallenges.indexOf(val) != -1 ? ++acc : acc;
+      }, 0);
+
+      progress += `${Math.round(100 * numCompleted / numChallenges)}%` + ',';
+    }
+  }
+
+  return progress;
+};
+
+// Writes the url and progress strings to a file
+let writeProgress = (writeArray, file) => {
+  const outputString = writeArray.join('\n');
+
+  fs.writeFile(file, outputString, (err) => {
+      // Report success / failure
+      const successMessage = 'Greenville Codes progress was written to ' + file;
+
+      err ? console.log(err) : console.log(successMessage);
+  }); 
+};
+
+// Runs the program
+let run = async () => {
+  // Get urls from urlFile
+  const urlArray = readURLs(urlFile);
+  let writeArray = [];
+
+  // Assemble student progress into array
+  for (let url of urlArray) {
+    let progress = await getHTML(url)
+                          .then(parseHTML)
+                          .then(getProgress);
+
+    writeArray[urlArray.indexOf(url)] = url + ',' + progress;
+  }
+
+  writeProgress(writeArray, outputFile);
+};
+
+run();

get_fcc_progress.js

@@ -0,0 +1,95 @@
+/**
+ * Scrapes a user profile from https://www.freecodecamp.org/{username} and 
+ * compares completed challenges against the FCC curriculum. The results are
+ * output in JSON format to outputFile.
+ * 
+ * Track Greenville Codes progress with gvl_codes_path.json.
+ * Track overall FreeCodeCamp progress with fcc_path.json.
+ * 
+ * Usage: node fcc_progress.js
+ * 
+ */
+
+const outputFile = 'fcc_progress.json',
+      url = 'https://www.freecodecamp.org/magoun',
+      trackFile = 'gvl_codes_path.json';
+
+const fs = require('fs');
+const puppeteer = require('puppeteer');
+
+// Fetch html using puppeteer
+let getHTML = async (url) => {
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+
+  // Wait for React components to load
+  await page.goto(url, { waitUntil: 'networkidle0'});
+
+  const html = await page.content();
+  await browser.close();
+  return html;
+};
+
+// Parse the HTML for completed challenges
+// Returns an array of challenge names
+let parseHTML = (html) => {
+  const $ = require('cheerio').load(html);
+  let challenges = [];
+
+  $('tr', 'tbody').each(function (index, element) {
+    let challenge = $(this).find('a').text();
+    challenges.push(challenge);
+  });
+
+  return challenges;
+};
+
+// Compare progress to overall curriculum
+// Returns a JSON object
+let getProgress = (completedChallenges) => {
+  const rawJSON = fs.readFileSync(trackFile);
+  let challengeMap = JSON.parse(rawJSON);
+
+  // Inconsistent capitalization, thus map to lower case
+  completedChallenges = completedChallenges.map(x => x.toLowerCase());
+
+  // Loop through curriculum JSON
+  for (let section of Object.keys(challengeMap)) {
+    for (let subsection of Object.keys(challengeMap[section])) {
+      let numChallenges = challengeMap[section][subsection].length;
+
+      // Inconsistent capitalization, thus map to lower case
+      let subsectionArray = challengeMap[section][subsection].map(x => x.toLowerCase());
+
+      let numCompleted = subsectionArray.reduce((acc, val) => {
+        return completedChallenges.indexOf(val) != -1 ? ++acc : acc;
+      }, 0);
+
+      challengeMap[section][subsection] = `${Math.round(100 * numCompleted / numChallenges)}%`;
+    }
+  }
+
+  return challengeMap;
+};
+
+// Writes the JSON object to a file
+let writeJSON = (json) => {
+  // Format the json for readability
+  const jsonString = JSON.stringify(json, null, 2);
+
+  fs.writeFile(outputFile, jsonString, (err) => {
+      // Report success / failure
+      const successMessage = 'Greenville Codes progress was written to ' + outputFile;
+      err ? console.log(err) : console.log(successMessage);
+  }); 
+};
+
+// Runs the program
+let run = () => {
+  getHTML(url)
+    .then(parseHTML)
+    .then(getProgress)
+    .then(writeJSON);
+};
+
+run();