From 0956525e942345e475e4e72a61846087e425fd4b Mon Sep 17 00:00:00 2001 From: Sriram Hariharan Date: Sun, 5 Mar 2023 14:34:26 -0600 Subject: [PATCH] auto-loading completely done --- src/shared/types/Course.ts | 6 +- src/views/components/CourseCatalogMain.tsx | 33 ++++---- .../common/Spinner/Spinner.module.scss | 23 ++++++ .../components/common/Spinner/Spinner.tsx | 14 ++++ .../injected/AutoLoad/AutoLoad.module.scss | 7 ++ .../components/injected/AutoLoad/AutoLoad.tsx | 73 +++++++++++++++++ .../components/injected/TableRow/TableRow.tsx | 3 +- src/views/hooks/useInfiniteScroll.ts | 39 ++++------ src/views/index.tsx | 2 +- src/views/lib/CourseCatalogScraper.ts | 10 +-- src/views/lib/getCourseTableRows.ts | 11 +++ src/views/lib/loadNextCourseCatalogPage.ts | 78 +++++++++++++++++++ todo.md | 2 +- 13 files changed, 248 insertions(+), 53 deletions(-) create mode 100644 src/views/components/common/Spinner/Spinner.module.scss create mode 100644 src/views/components/common/Spinner/Spinner.tsx create mode 100644 src/views/components/injected/AutoLoad/AutoLoad.module.scss create mode 100644 src/views/components/injected/AutoLoad/AutoLoad.tsx create mode 100644 src/views/lib/getCourseTableRows.ts create mode 100644 src/views/lib/loadNextCourseCatalogPage.ts diff --git a/src/shared/types/Course.ts b/src/shared/types/Course.ts index f0fa8dc5..075c2268 100644 --- a/src/shared/types/Course.ts +++ b/src/shared/types/Course.ts @@ -78,9 +78,9 @@ export class Course { } /** - * A helper type that is used to represent a row in the course schedule table, with the actual element corresponding to the course object + * A helper type that is used to represent an element in the DOM, with the actual element corresponding to the course object */ -export type CourseRow = { - rowElement: HTMLTableRowElement; +export type ScrapedRow = { + element: HTMLTableRowElement; course: Course; }; diff --git a/src/views/components/CourseCatalogMain.tsx b/src/views/components/CourseCatalogMain.tsx index b31b9cf7..0a54eaa9 100644 --- a/src/views/components/CourseCatalogMain.tsx +++ b/src/views/components/CourseCatalogMain.tsx @@ -1,11 +1,12 @@ import React, { useEffect, useState } from 'react'; -import { Course, CourseRow } from 'src/shared/types/Course'; -import useInfiniteScroll from '../hooks/useInfiniteScroll'; +import { Course, ScrapedRow } from 'src/shared/types/Course'; import { useKeyPress } from '../hooks/useKeyPress'; import { CourseCatalogScraper } from '../lib/CourseCatalogScraper'; +import getCourseTableRows from '../lib/getCourseTableRows'; import { SiteSupport } from '../lib/getSiteSupport'; import { populateSearchInputs } from '../lib/populateSearchInputs'; import ExtensionRoot from './common/ExtensionRoot/ExtensionRoot'; +import AutoLoad from './injected/AutoLoad/AutoLoad'; import CoursePanel from './injected/CoursePanel/CoursePanel'; import TableHead from './injected/TableHead'; import TableRow from './injected/TableRow/TableRow'; @@ -18,24 +19,26 @@ interface Props { * This is the top level react component orchestrating the course catalog page. */ export default function CourseCatalogMain({ support }: Props) { - const [rows, setRows] = React.useState([]); + const [rows, setRows] = React.useState([]); const [selectedCourse, setSelectedCourse] = useState(null); - const isScrolling = useInfiniteScroll(async () => { - console.log('infinite scroll'); - return true; - }); - useEffect(() => { populateSearchInputs(); }, []); useEffect(() => { - const scraper = new CourseCatalogScraper(support); - const rows = scraper.scrape(document.querySelectorAll('table tbody tr')); - console.log('useEffect -> rows:', rows); - setRows(rows); - }, []); + const tableRows = getCourseTableRows(document); + const ccs = new CourseCatalogScraper(support); + const scrapedRows = ccs.scrape(tableRows); + setRows(scrapedRows); + }, [support]); + + const addRows = (newRows: ScrapedRow[]) => { + newRows.forEach(row => { + document.querySelector('table tbody')!.appendChild(row.element); + }); + setRows([...rows, ...newRows]); + }; const handleRowButtonClick = (course: Course) => () => { setSelectedCourse(course); @@ -52,7 +55,7 @@ export default function CourseCatalogMain({ support }: Props) { Plus {rows.map(row => ( ))} {selectedCourse && } - {isScrolling &&
Scrolling...
} + ); } diff --git a/src/views/components/common/Spinner/Spinner.module.scss b/src/views/components/common/Spinner/Spinner.module.scss new file mode 100644 index 00000000..6267bb9b --- /dev/null +++ b/src/views/components/common/Spinner/Spinner.module.scss @@ -0,0 +1,23 @@ +@import 'src/views/styles/base.module.scss'; + +$spinner-border-width: 10px; + +.spinner { + border: 1px solid $CHARCOAL; + border-width: $spinner-border-width; + border-top: $spinner-border-width solid $TANGERINE; + margin: 0 auto 15px auto; + border-radius: 50%; + width: 60px; + height: 60px; + animation: spin 1s linear infinite; +} + +@keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} diff --git a/src/views/components/common/Spinner/Spinner.tsx b/src/views/components/common/Spinner/Spinner.tsx new file mode 100644 index 00000000..4ef5fa2c --- /dev/null +++ b/src/views/components/common/Spinner/Spinner.tsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { ISassColors } from 'src/views/styles/colors.module.scss'; +import styles from './Spinner.module.scss'; + +type Props = { + color?: keyof ISassColors; +}; + +/** + * A simple spinner component that can be used to indicate loading. + */ +export default function Spinner({ color }: Props) { + return
; +} diff --git a/src/views/components/injected/AutoLoad/AutoLoad.module.scss b/src/views/components/injected/AutoLoad/AutoLoad.module.scss new file mode 100644 index 00000000..db7d5f02 --- /dev/null +++ b/src/views/components/injected/AutoLoad/AutoLoad.module.scss @@ -0,0 +1,7 @@ +@import 'src/views/styles/base.module.scss'; + +.autoLoad { + display: flex; + align-items: center; + justify-content: center; +} diff --git a/src/views/components/injected/AutoLoad/AutoLoad.tsx b/src/views/components/injected/AutoLoad/AutoLoad.tsx new file mode 100644 index 00000000..30fe8727 --- /dev/null +++ b/src/views/components/injected/AutoLoad/AutoLoad.tsx @@ -0,0 +1,73 @@ +import React, { useEffect, useState } from 'react'; +import { createPortal } from 'react-dom'; +import { ScrapedRow } from 'src/shared/types/Course'; +import useInfiniteScroll from 'src/views/hooks/useInfiniteScroll'; +import { CourseCatalogScraper } from 'src/views/lib/CourseCatalogScraper'; +import { SiteSupport } from 'src/views/lib/getSiteSupport'; +import { loadNextCourseCatalogPage, AutoLoadStatus } from 'src/views/lib/loadNextCourseCatalogPage'; +import Spinner from '../../common/Spinner/Spinner'; +import styles from './AutoLoad.module.scss'; + +type Props = { + addRows: (rows: ScrapedRow[]) => void; +}; + +/** + * This component is responsible for loading the next page of courses when the user scrolls to the bottom of the page. + * @returns + */ +export default function AutoLoad({ addRows }: Props) { + const [container, setContainer] = useState(null); + const [status, setStatus] = useState(AutoLoadStatus.IDLE); + + useEffect(() => { + const portalContainer = document.createElement('div'); + const lastTableCell = document.querySelector('table')!; + lastTableCell!.after(portalContainer); + setContainer(portalContainer); + }, []); + + // FOR DEBUGGING + useEffect(() => { + console.log(`AutoLoad is now ${status}`); + }, [status]); + + // This hook will call the callback when the user scrolls to the bottom of the page. + useInfiniteScroll(async () => { + // fetch the next page of courses + const [status, nextRows] = await loadNextCourseCatalogPage(); + setStatus(status); + if (!nextRows) { + return; + } + // scrape the courses from the page + const ccs = new CourseCatalogScraper(SiteSupport.COURSE_CATALOG_LIST); + const scrapedRows = ccs.scrape(nextRows); + + // add the scraped courses to the current page + addRows(scrapedRows); + }, [addRows]); + + if (!container || status === AutoLoadStatus.IDLE) { + return null; + } + + return createPortal( +
+ {status === AutoLoadStatus.LOADING && ( +
+ +

Loading Next Page...

+
+ )} + {status === AutoLoadStatus.ERROR && ( +
+

Something went wrong

+

Try refreshing the page

+ +
+ )} +
, + container + ); +} diff --git a/src/views/components/injected/TableRow/TableRow.tsx b/src/views/components/injected/TableRow/TableRow.tsx index 3e0a3914..34f3af99 100644 --- a/src/views/components/injected/TableRow/TableRow.tsx +++ b/src/views/components/injected/TableRow/TableRow.tsx @@ -1,6 +1,6 @@ import React, { useEffect, useState } from 'react'; import ReactDOM from 'react-dom'; -import { Course, CourseRow } from 'src/shared/types/Course'; +import { Course, ScrapedRow } from 'src/shared/types/Course'; import { SiteSupport } from 'src/views/lib/getSiteSupport'; import { Button } from '../../common/Button/Button'; import styles from './TableRow.module.scss'; @@ -18,7 +18,6 @@ interface Props { * @returns a react portal to the new td in the column or null if the column has not been created yet. */ export default function TableRow({ support, course, element, isSelected, onClick }: Props): JSX.Element | null { - console.log('TableRow -> isSelected:', isSelected); const [container, setContainer] = useState(null); useEffect(() => { diff --git a/src/views/hooks/useInfiniteScroll.ts b/src/views/hooks/useInfiniteScroll.ts index 333591ef..2c31017e 100644 --- a/src/views/hooks/useInfiniteScroll.ts +++ b/src/views/hooks/useInfiniteScroll.ts @@ -1,5 +1,4 @@ -import { useState, useEffect } from 'react'; -import { sleep } from 'src/shared/util/time'; +import { useEffect } from 'react'; /** * Hook to execute a callback when the user scrolls to the bottom of the page @@ -7,32 +6,20 @@ import { sleep } from 'src/shared/util/time'; * @returns isLoading boolean to indicate if the callback is currently being executed */ -export default function useInfiniteScroll(callback: () => Promise): boolean { - const [isLoading, setIsLoading] = useState(false); +export default function useInfiniteScroll( + callback: () => Promise | void, + deps?: React.DependencyList | undefined +) { + const isScrolling = () => { + const { innerHeight } = window; + const { scrollTop, offsetHeight } = document.documentElement; + if (innerHeight + scrollTop >= offsetHeight) { + callback(); + } + }; useEffect(() => { window.addEventListener('scroll', isScrolling); return () => window.removeEventListener('scroll', isScrolling); - }, []); - - useEffect(() => { - if (!isLoading) return; - callback().then(isFinished => { - if (!isFinished) { - sleep(1000).then(() => { - setIsLoading(false); - }); - } - }); - }, [isLoading]); - - function isScrolling() { - if ( - window.innerHeight + document.documentElement.scrollTop !== document.documentElement.offsetHeight || - isLoading - ) - return; - setIsLoading(true); - } - return isLoading; + }, deps); } diff --git a/src/views/index.tsx b/src/views/index.tsx index 61cca788..6b4baaf1 100644 --- a/src/views/index.tsx +++ b/src/views/index.tsx @@ -16,7 +16,7 @@ if (isExtensionPopup()) { } if (support === SiteSupport.COURSE_CATALOG_DETAILS || support === SiteSupport.COURSE_CATALOG_LIST) { - const shadowDom = createShadowDOM('ut-registration-plus-dom-container'); + const shadowDom = createShadowDOM('ut-registration-plus-container'); render(, shadowDom.shadowRoot); shadowDom.addStyle('static/css/content.css'); } diff --git a/src/views/lib/CourseCatalogScraper.ts b/src/views/lib/CourseCatalogScraper.ts index 0dee546e..6a79ad10 100644 --- a/src/views/lib/CourseCatalogScraper.ts +++ b/src/views/lib/CourseCatalogScraper.ts @@ -1,4 +1,4 @@ -import { Course, Instructor, Status, InstructionMode, CourseRow } from 'src/shared/types/Course'; +import { Course, Instructor, Status, InstructionMode, ScrapedRow } from 'src/shared/types/Course'; import { CourseSchedule, CourseMeeting } from 'src/shared/types/CourseSchedule'; import { SiteSupport } from 'src/views/lib/getSiteSupport'; @@ -27,7 +27,7 @@ enum DetailsSelector { } /** - * A class that allows use to scrape information from UT's course catalog to create our internal representation of a course + * A class that allows us to scrape information from UT's course catalog to create our internal representation of a course */ export class CourseCatalogScraper { support: SiteSupport; @@ -41,8 +41,8 @@ export class CourseCatalogScraper { * @param rows the rows of the course catalog table * @returns an array of course row objects (which contain courses corresponding to the htmltable row) */ - public scrape(rows: NodeListOf): CourseRow[] { - const courses: CourseRow[] = []; + public scrape(rows: NodeListOf | HTMLTableRowElement[]): ScrapedRow[] { + const courses: ScrapedRow[] = []; let fullName = this.getFullName(); @@ -94,7 +94,7 @@ export class CourseCatalogScraper { }, }); courses.push({ - rowElement: row, + element: row, course: newCourse, }); }); diff --git a/src/views/lib/getCourseTableRows.ts b/src/views/lib/getCourseTableRows.ts new file mode 100644 index 00000000..b1a91041 --- /dev/null +++ b/src/views/lib/getCourseTableRows.ts @@ -0,0 +1,11 @@ +const TABLE_ROW_SELECTOR = 'table tbody tr'; + +/** + * Returns an array of all the rows in the course table on the passed in document + * @param doc the document to get the course table rows from + * @returns an array of all the rows in the course table on the passed in document + */ +export default function getCourseTableRows(doc: Document): HTMLTableRowElement[] { + const courseRows = doc.querySelectorAll(TABLE_ROW_SELECTOR); + return Array.from(courseRows); +} diff --git a/src/views/lib/loadNextCourseCatalogPage.ts b/src/views/lib/loadNextCourseCatalogPage.ts new file mode 100644 index 00000000..2faf5bab --- /dev/null +++ b/src/views/lib/loadNextCourseCatalogPage.ts @@ -0,0 +1,78 @@ +import getCourseTableRows from './getCourseTableRows'; + +const NEXT_PAGE_BUTTON_SELECTOR = '#next_nav_link'; +const PREV_PAGE_BUTTON_SELECTOR = '#prev_nav_link'; +/** + * Represents all the states that we care about when autoloading the next page of courses + */ +export enum AutoLoadStatus { + LOADING = 'LOADING', + IDLE = 'IDLE', + ERROR = 'ERROR', +} + +let isLoading = false; +let nextPageURL = getNextButton(document)?.href; + +/** + * This will scrape the pagination buttons from the course list and use them to load the next page + * and then return the table rows from the next page + * @returns a tuple of the current LoadStatus (whether are currently loading the next page, or if we have reached the end of the course catalog, + * or if there was an error loading the next page) and an array of the table rows from the next page (or an empty array + * if we have reached the end of the course catalog + */ +export async function loadNextCourseCatalogPage(): Promise<[AutoLoadStatus, HTMLTableRowElement[]]> { + // if there is no more nextPageURL, then we have reached the end of the course catalog, so we can stop + if (!nextPageURL) { + return [AutoLoadStatus.IDLE, []]; + } + // remove the next button so that we don't load the same page twice + removePaginationButtons(document); + if (isLoading) { + // if we are already loading the next page, then we don't need to do anything + return [AutoLoadStatus.LOADING, []]; + } + + // begin loading the next page + isLoading = true; + try { + const response = await fetch(nextPageURL); + const html = await response.text(); + const parser = new DOMParser(); + const newDocument = parser.parseFromString(html, 'text/html'); + + // extract the table rows from the document of the next page + const tableRows = getCourseTableRows(newDocument); + if (!tableRows) { + return [AutoLoadStatus.ERROR, []]; + } + + // extract the next page url from the document of the next page, so when we scroll again we can use that + nextPageURL = getNextButton(newDocument)?.href; + isLoading = false; + return [AutoLoadStatus.IDLE, Array.from(tableRows)]; + } catch (e) { + console.error(e); + return [AutoLoadStatus.ERROR, []]; + } +} + +/** + * Scrapes the next button from the document + * @param doc the document to get the next button from + * @returns the next button from the document + */ +function getNextButton(doc: Document) { + return doc.querySelector(NEXT_PAGE_BUTTON_SELECTOR); +} + +/** + * Removes the next and previous buttons from the document so that we don't load the same page twice + * @param doc the document to remove the next and previous buttons from + */ +export function removePaginationButtons(doc: Document) { + const nextButton = doc.querySelectorAll(NEXT_PAGE_BUTTON_SELECTOR); + nextButton.forEach(button => button.remove()); + const prevButton = doc.querySelectorAll(PREV_PAGE_BUTTON_SELECTOR); + prevButton.forEach(button => button.remove()); +} diff --git a/todo.md b/todo.md index c5ded611..32f15724 100644 --- a/todo.md +++ b/todo.md @@ -6,7 +6,7 @@ Last Updated: 03/4/2023 - [x] scraping course information - [x] injecting plus header and buttons -- [ ] auto loading next pages +- [x] auto loading next pages - [ ] showing course popup - [ ] RMP, eCis, Textbook, and Syllabus buttons - [ ] displaying professor information on popup