query grade distributions working, and filtering by semesters working

package.json

@@ -13,6 +13,7 @@
         "lint": "eslint ./ --ext .ts,.tsx"
     },
     "dependencies": {
+        "@types/sql.js": "^1.4.4",
         "chrome-extension-toolkit": "^0.0.23",
         "classnames": "^2.3.2",
         "clean-webpack-plugin": "^4.0.0",
@@ -21,6 +22,7 @@
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "sass": "^1.57.1",
+        "sql.js": "1.8.0",
         "uuid": "^9.0.0"
     },
     "devDependencies": {
@@ -40,6 +42,7 @@
         "conventional-changelog-conventionalcommits": "^5.0.0",
         "copy-webpack-plugin": "^11.0.0",
         "create-file-webpack": "^1.0.2",
+        "crypto-browserify": "^3.12.0",
         "css-loader": "^6.7.3",
         "dotenv": "^16.0.3",
         "esbuild-loader": "^2.20.0",
@@ -69,6 +72,7 @@
         "simple-git": "^3.15.1",
         "socket.io": "^4.5.4",
         "socket.io-client": "^4.5.4",
+        "stream-browserify": "^3.0.0",
         "terser-webpack-plugin": "^5.3.6",
         "ts-node": "^10.9.1",
         "tsconfig-paths-webpack-plugin": "^4.0.0",

src/background/background.ts

@@ -9,7 +9,6 @@ import { SessionStore } from './storage/SessionStore';
 import browserActionHandler from './handler/browserActionHandler';
 import hotReloadingHandler from './handler/hotReloadingHandler';
 import tabManagementHandler from './handler/tabManagementHandler';
-import courseDataHandler from './handler/courseDataHandler';
 
 onServiceWorkerAlive();
 
@@ -35,7 +34,6 @@ const messageListener = new MessageListener<BACKGROUND_MESSAGES>({
     ...browserActionHandler,
     ...hotReloadingHandler,
     ...tabManagementHandler,
-    ...courseDataHandler,
 });
 
 messageListener.listen();

src/background/handler/courseDataHandler.ts

@@ -1,14 +0,0 @@
-import { MessageHandler } from 'chrome-extension-toolkit';
-import CourseDataMessages from 'src/shared/messages/CourseDataMessages';
-
-const courseDataHandler: MessageHandler<CourseDataMessages> = {
-    getDistribution({ data, sendResponse }) {
-        const { course } = data;
-
-        const dummyData = Array.from({ length: 12 }, () => Math.floor(Math.random() * 100));
-
-        sendResponse(dummyData);
-    },
-};
-
-export default courseDataHandler;

src/shared/messages/CourseDataMessages.ts

@@ -1,9 +0,0 @@
-import { Course } from '../types/Course';
-
-export default interface CourseDataMessages {
-    /**
-     * Gets the distribution of grades for the given course
-     * @returns an array of the number of students in each grade range from A+ to F, with the index of the array corresponding to the grade range
-     */
-    getDistribution: (data: { course: Course }) => number[] | undefined;
-}

src/shared/messages/index.ts

@@ -3,15 +3,11 @@ import TAB_MESSAGES from './TabMessages';
 import BrowserActionMessages from './BrowserActionMessages';
 import HotReloadingMessages from './HotReloadingMessages';
 import TabManagementMessages from './TabManagementMessages';
-import CourseDataMessages from './CourseDataMessages';
 
 /**
  * This is a type with all the message definitions that can be sent TO the background script
  */
-export type BACKGROUND_MESSAGES = BrowserActionMessages &
+export type BACKGROUND_MESSAGES = BrowserActionMessages & TabManagementMessages & HotReloadingMessages;
-    TabManagementMessages &
-    HotReloadingMessages &
-    CourseDataMessages;
 
 /**
  * A utility object that can be used to send type-safe messages to the background script

src/shared/types/Course.ts

@@ -27,7 +27,7 @@ export type Semester = {
     /** The season that the semester is in (Fall, Spring, Summer) */
     season: 'Fall' | 'Spring' | 'Summer';
     /** UT's code for the semester */
-    code: string;
+    code?: string;
 };
 
 /**

src/shared/types/Distribution.ts

@@ -0,0 +1,40 @@
+/* eslint-disable max-classes-per-file */
+import { PointOptionsObject } from 'highcharts';
+import { Semester } from './Course';
+/**
+ * Each of the possible letter grades that can be given in a course
+ */
+export type LetterGrade = 'A' | 'A-' | 'B' | 'B+' | 'B-' | 'C' | 'C+' | 'C-' | 'D' | 'D+' | 'D-' | 'F';
+
+/**
+ * A distribution of grades for a course,
+ * with the keys being the letter grades and the values being the number of students that got that grade
+ */
+export type Distribution = {
+    [key in LetterGrade]: number;
+};
+
+/**
+ * This is a object-ified version of a row in the SQL table that is used to store the distribution data.
+ */
+export type CourseSQLRow = {
+    sem?: string;
+    prof?: string;
+    dept?: string;
+    course_nbr?: string;
+    course_name?: string;
+    a1?: number;
+    a2?: number;
+    a3?: number;
+    b1?: number;
+    b2?: number;
+    b3?: number;
+    c1?: number;
+    c2?: number;
+    c3?: number;
+    d1?: number;
+    d2?: number;
+    d3?: number;
+    f?: number;
+    semesters?: string;
+};

src/views/components/common/Button/Button.tsx

@@ -27,7 +27,7 @@ export function Button({
     return (
         <button
             style={style}
-            data-testId={testId}
+            data-testid={testId}
             className={classNames(styles.button, className, styles[type ?? 'primary'], {
                 [styles.disabled]: disabled,
             })}

src/views/components/common/Spinner/Spinner.tsx

@@ -12,5 +12,5 @@ type Props = {
  * A simple spinner component that can be used to indicate loading.
  */
 export default function Spinner({ className, testId, style }: Props) {
-    return <div data-testId={testId} style={style} className={classNames(styles.spinner, className)} />;
+    return <div data-testid={testId} style={style} className={classNames(styles.spinner, className)} />;
 }

src/views/components/injected/CoursePopup/CourseDescription/CourseDescription.tsx

@@ -44,8 +44,8 @@ export default function CourseDescription({ course }: Props) {
             {status === LoadStatus.DONE && (
                 <ul className={styles.description}>
                     {description.map(paragraph => (
-                        <li>
+                        <li key={paragraph}>
-                            <DescriptionLine key={paragraph} line={paragraph} />
+                            <DescriptionLine line={paragraph} />
                         </li>
                     ))}
                 </ul>
@@ -64,9 +64,7 @@ function DescriptionLine({ line }: LineProps) {
     const className = classNames({
         [styles.prerequisite]: lowerCaseLine.includes('prerequisite'),
         [styles.onlyOne]:
-            lowerCaseLine.includes('may be') ||
+            lowerCaseLine.includes('may be') || lowerCaseLine.includes('only one') || lowerCaseLine.includes('may not'),
-            lowerCaseLine.includes('only one') ||
-            lowerCaseLine.includes('may not be'),
         [styles.restriction]: lowerCaseLine.includes('restrict'),
     });
 

src/views/components/injected/CoursePopup/CoursePopup.module.scss

@@ -1,7 +1,7 @@
 .popup {
     border-radius: 12px;
     position: relative;
-    max-width: 50%;
+    width: 55%;
     overflow-y: auto;
     max-height: 90%;
 

src/views/components/injected/CoursePopup/CoursePopup.tsx

@@ -1,6 +1,5 @@
 import React from 'react';
 import { Course } from 'src/shared/types/Course';
-import Card from '../../common/Card/Card';
 import Popup from '../../common/Popup/Popup';
 import CourseDescription from './CourseDescription/CourseDescription';
 import CourseHeader from './CourseHeader/CourseHeader';

src/views/components/injected/CoursePopup/GradeDistribution/GradeDistribution.tsx

@@ -3,12 +3,17 @@ import React, { useEffect, useRef, useState } from 'react';
 import HighchartsReact from 'highcharts-react-official';
 import Highcharts from 'highcharts';
 import Card from 'src/views/components/common/Card/Card';
-import { bMessenger } from 'src/shared/messages';
+import { Course, Semester } from 'src/shared/types/Course';
-import { Course } from 'src/shared/types/Course';
 import colors from 'src/views/styles/colors.module.scss';
 import Spinner from 'src/views/components/common/Spinner/Spinner';
 import Text from 'src/views/components/common/Text/Text';
 import Icon from 'src/views/components/common/Icon/Icon';
+import { Distribution, LetterGrade } from 'src/shared/types/Distribution';
+import {
+    NoDataError,
+    queryAggregateDistribution,
+    querySemesterDistribution,
+} from 'src/views/lib/database/queryDistribution';
 import styles from './GradeDistribution.module.scss';
 
 enum DataStatus {
@@ -22,12 +27,32 @@ interface Props {
     course: Course;
 }
 
+const GRADE_COLORS: Record<LetterGrade, string> = {
+    A: colors.turtle_pond,
+    'A-': colors.turtle_pond,
+    'B+': colors.cactus,
+    B: colors.cactus,
+    'B-': colors.cactus,
+    'C+': colors.sunshine,
+    C: colors.sunshine,
+    'C-': colors.sunshine,
+    'D+': colors.tangerine,
+    D: colors.tangerine,
+    'D-': colors.tangerine,
+    F: colors.speedway_brick,
+};
+
 /**
  * A chart to fetch and display the grade distribution for a course
  * @returns
  */
 export default function GradeDistribution({ course }: Props) {
     const ref = useRef<HighchartsReact.RefObject>(null);
+    const [semesters, setSemesters] = useState<Semester[]>([]);
+    const [selectedSemester, setSelectedSemester] = useState<Semester | null>(null);
+    const [distribution, setDistribution] = useState<Distribution | null>(null);
+    const [status, setStatus] = useState<DataStatus>(DataStatus.LOADING);
+
     const [chartOptions, setChartOptions] = useState<Highcharts.Options>({
         title: {
             text: undefined,
@@ -89,48 +114,79 @@ export default function GradeDistribution({ course }: Props) {
             },
         ],
     });
-    const [status, setStatus] = useState<DataStatus>(DataStatus.LOADING);
 
-    useEffect(() => {
+    const updateChart = (distribution: Distribution) => {
-        bMessenger.getDistribution({ course }).then(distribution => {
-            if (!distribution) {
-                return setStatus(DataStatus.ERROR);
-            }
-            if (!distribution.length) {
-                return setStatus(DataStatus.NOT_FOUND);
-            }
         setChartOptions(options => ({
             ...options,
             series: [
                 {
                     type: 'column',
                     name: 'Grades',
-                        data: distribution.map((y, i) => ({
+                    data: Object.entries(distribution).map(([grade, count]) => ({
-                            y,
+                        y: count,
-                            color:
+                        color: GRADE_COLORS[grade as LetterGrade],
-                                i < 3
-                                    ? colors.turtle_pond
-                                    : i < 5
-                                    ? colors.cactus
-                                    : i < 7
-                                    ? colors.sunshine
-                                    : i < 10
-                                    ? colors.tangerine
-                                    : colors.speedway_brick,
                     })),
                 },
             ],
         }));
-            setStatus(DataStatus.FOUND);
-            // the highcharts library kinda sucks and doesn't resize the chart when the window resizes,
-            // so we have to manually trigger a resize event when the chart is rendered 🙃
         window.dispatchEvent(new Event('resize'));
+    };
+
+    useEffect(() => {
+        queryAggregateDistribution(course)
+            .then(([distribution, semesters]) => {
+                setSemesters(semesters);
+                updateChart(distribution);
+                setStatus(DataStatus.FOUND);
+            })
+            .catch(err => {
+                if (err instanceof NoDataError) {
+                    return setStatus(DataStatus.NOT_FOUND);
+                }
+                return setStatus(DataStatus.ERROR);
             });
     }, [course]);
 
+    useEffect(() => {
+        (async () => {
+            let distribution: Distribution;
+            if (selectedSemester) {
+                distribution = await querySemesterDistribution(course, selectedSemester);
+            } else {
+                [distribution] = await queryAggregateDistribution(course);
+            }
+            updateChart(distribution);
+            setStatus(DataStatus.FOUND);
+        })().catch(err => {
+            if (err instanceof NoDataError) {
+                return setStatus(DataStatus.NOT_FOUND);
+            }
+            return setStatus(DataStatus.ERROR);
+        });
+    }, [selectedSemester, course]);
+
+    const handleSelectSemester = (event: React.ChangeEvent<HTMLSelectElement>) => {
+        const index = parseInt(event.target.value, 10);
+        if (index === 0) {
+            setSelectedSemester(null);
+        } else {
+            setSelectedSemester(semesters[index - 1]);
+        }
+    };
+
     if (status === DataStatus.FOUND) {
         return (
             <Card className={styles.chartContainer}>
+                {semesters.length > 0 && (
+                    <select onChange={handleSelectSemester}>
+                        <option value={0}>Aggregate</option>
+                        {semesters.map((semester, index) => (
+                            <option key={semester.season + semester.year} value={index + 1}>
+                                {semester.season} {semester.year}
+                            </option>
+                        ))}
+                    </select>
+                )}
                 <HighchartsReact ref={ref} highcharts={Highcharts} options={chartOptions} />
             </Card>
         );

src/views/lib/database/initializeDB.ts

@@ -0,0 +1,35 @@
+import initSqlJs from 'sql.js/dist/sql-wasm';
+
+const WASM_FILE_URL = chrome.runtime.getURL('database/sql-wasm.wasm');
+const DB_FILE_URL = chrome.runtime.getURL('database/grades.db');
+
+/**
+ * A utility type for the SQL.js Database type
+ */
+export type Database = initSqlJs.Database;
+
+/**
+ * We only want to load the database into memory once, so we store a reference to the database here.
+ */
+let db: Database;
+
+/**
+ * This function loads the database into memory and returns a reference to the sql.js Database object.
+ * @returns a reference to the sql.js Database object
+ */
+export async function initializeDB(): Promise<Database> {
+    if (!WASM_FILE_URL || !DB_FILE_URL) {
+        throw new Error('WASM_FILE_URL or DB_FILE_URL is undefined');
+    }
+    if (db) {
+        return db;
+    }
+    const { Database } = await initSqlJs({
+        locateFile: file => WASM_FILE_URL,
+    });
+
+    const dbBuffer = await fetch(DB_FILE_URL).then(res => res.arrayBuffer());
+    db = new Database(new Uint8Array(dbBuffer));
+
+    return db;
+}

src/views/lib/database/queryDistribution.ts

@@ -0,0 +1,115 @@
+import { Course, Semester } from 'src/shared/types/Course';
+import { CourseSQLRow, Distribution } from 'src/shared/types/Distribution';
+import { initializeDB } from './initializeDB';
+
+/**
+ * fetches the aggregate distribution of grades for a given course from the course db, and the semesters that we have data for
+ * @param course the course to fetch the distribution for
+ * @returns a Distribution object containing the distribution of grades for the course, and
+ * an array of semesters that we have the distribution for
+ */
+export async function queryAggregateDistribution(course: Course): Promise<[Distribution, Semester[]]> {
+    const db = await initializeDB();
+    const query = generateQuery(course, null);
+
+    const res = db.exec(query)?.[0];
+    if (!res?.columns?.length) {
+        throw new NoDataError(course);
+    }
+
+    let row: Required<CourseSQLRow> = {} as Required<CourseSQLRow>;
+    res.columns.forEach((col, i) => {
+        row[res.columns[i]] = res.values[0][i];
+    });
+
+    const distribution: Distribution = {
+        A: row.a2,
+        'A-': row.a3,
+        'B+': row.b1,
+        B: row.b2,
+        'B-': row.b3,
+        'C+': row.c1,
+        C: row.c2,
+        'C-': row.c3,
+        'D+': row.d1,
+        D: row.d2,
+        'D-': row.d3,
+        F: row.f,
+    };
+
+    const semesters: Semester[] = row.semesters.split(',').map((sem: string) => {
+        const [season, year] = sem.split(' ');
+        return { year: parseInt(year, 10), season: season as Semester['season'] };
+    });
+
+    return [distribution, semesters];
+}
+
+/**
+ * Creates a SQL query that we can execute on the database to get the distribution of grades for a given course in a given semester
+ * @param course the course to fetch the distribution for
+ * @param semester the semester to fetch the distribution for OR null if we want the aggregate distribution
+ * @returns a SQL query string
+ */
+function generateQuery(course: Course, semester: Semester | null): string {
+    const profName = course.instructors[0]?.fullName;
+
+    const query = `
+        select * from ${semester ? 'grades' : 'agg'}
+        where dept like '%${course.department}%'
+        ${profName ? `and prof like '%${profName}%'` : ''}
+        and course_nbr like '%${course.number}%'
+        ${semester ? `and sem like '%${semester.season} ${semester.year}%'` : ''}
+        order by a1+a2+a3+b1+b2+b3+c1+c2+c3+d1+d2+d3+f desc
+    `;
+
+    return query;
+}
+
+/**
+ * fetches the distribution of grades for a semester for a given course from the course db
+ * @param course the course to fetch the distribution for
+ * @param semester the semester to fetch the distribution for
+ * @returns a Distribution object containing the distribution of grades for the course
+ */
+export async function querySemesterDistribution(course: Course, semester: Semester): Promise<Distribution> {
+    const db = await initializeDB();
+    const query = generateQuery(course, semester);
+
+    const res = db.exec(query)?.[0];
+    if (!res?.columns?.length) {
+        throw new NoDataError(course);
+    }
+
+    let row: Required<CourseSQLRow> = {} as Required<CourseSQLRow>;
+    res.columns.forEach((col, i) => {
+        row[res.columns[i]] = res.values[0][i];
+    });
+
+    const distribution: Distribution = {
+        A: row.a2,
+        'A-': row.a3,
+        'B+': row.b1,
+        B: row.b2,
+        'B-': row.b3,
+        'C+': row.c1,
+        C: row.c2,
+        'C-': row.c3,
+        'D+': row.d1,
+        D: row.d2,
+        'D-': row.d3,
+        F: row.f,
+    };
+
+    return distribution;
+}
+
+/**
+ * A custom error class for when we don't have data for a course
+ */
+export class NoDataError extends Error {
+    constructor(course: Course) {
+        super(`No data for #${course.uniqueId} ${course.department} ${course.number}`);
+        this.name = 'NoDataError';
+    }
+}

webpack/webpack.config.ts

@@ -60,6 +60,7 @@ export default function config(mode: Environment, manifest: chrome.runtime.Manif
                 crypto: 'crypto-browserify',
                 stream: 'stream-browserify',
                 buffer: 'buffer',
+                fs: false,
             },
         },
         // this is where we define the loaders for different file types