ahbeng/NUSMods

View on GitHub
scrapers/nus-v2/src/index.ts

Summary

Maintainability
A
0 mins
Test Coverage
// Imported for side effects, so they have to be right at the top
import './utils/sentry';

import * as yargs from 'yargs';
import { size, mapValues } from 'lodash';

import logger from './services/logger';
import { Aliases, Semesters } from './types/modules';

import TestApi from './tasks/TestApi';
import GetFacultyDepartment from './tasks/GetFacultyDepartment';
import GetSemesterData from './tasks/GetSemesterData';
import CollateVenues from './tasks/CollateVenues';
import CollateModules from './tasks/CollateModules';
import DataPipeline from './tasks/DataPipeline';

import config from './config';
import MigrateV1ToV2 from './tasks/MigrateV1ToV2';

function handleFatalError(e: Error): void {
  logger.fatal(e, 'Fatal error');
  // Do not use process.exit because that will quit without waiting for the
  // event loop to empty, which may cut off logs or IO operations
  process.exitCode = 1;
}

/* eslint-disable @typescript-eslint/no-explicit-any */
function run(fn: (...args: any[]) => Promise<any>) {
  return (...args: any[]) => fn(...args).catch(handleFatalError);
}
/* eslint-enable */

const parameters: Record<string, yargs.Options> = {
  sem: {
    type: 'number',
    choices: Semesters,
  },
  year: {
    type: 'string',
    default: config.academicYear,
    coerce: (value: string) => {
      // Handle year given in two or four number form
      if (value.length === 2) return `20${value}/20${+value + 1}`;
      if (value.length === 4) return `${value}-${+value + 1}`;
      return value.replace('-', '/');
    },
  },
};

/* eslint-disable no-await-in-loop */

// eslint-disable-next-line no-unused-expressions
yargs
  .command({
    command: 'test',
    describe: 'run some simple tests against the API to ensure things are set up correctly',
    handler: run(() => new TestApi(config.academicYear).run()),
  })
  .command({
    command: 'departments [year]',
    aliases: ['department', 'faculty', 'faculties'],
    describe: 'download data for all active departments and faculties',
    builder: {
      year: parameters.year,
    },
    handler: run(({ year }) => new GetFacultyDepartment(year).run()),
  })
  .command({
    command: 'semester <sem> [year]',
    describe: 'download all data for the given semester',
    builder: {
      sem: parameters.sem,
      year: parameters.year,
    },
    handler: run(async ({ sem, year }) => {
      const organizations = await new GetFacultyDepartment(year).run();
      const modules = await new GetSemesterData(sem).run(organizations);
      logger.info(`Collected data for ${modules.length} modules`);
    }),
  })
  .command({
    command: 'venue [year] <sem>',
    aliases: ['venues'],
    describe: 'collate venue for given semester',
    builder: {
      sem: parameters.sem,
      year: parameters.year,
    },
    handler: run(async ({ sem, year }) => {
      const modules = await new GetSemesterData(sem, year).outputCache.read();
      const { venues } = await new CollateVenues(sem, year).run(modules);
      logger.info(`Collated ${size(venues)} venues`);
    }),
  })
  .command({
    command: 'combine [year]',
    describe: 'combine semester data for modules',
    builder: {
      year: parameters.year,
    },
    handler: run(async ({ year }) => {
      const aliases = [];
      const semesterData = [];

      for (const semester of Semesters) {
        const semesterAliases = await new CollateVenues(semester, year).aliasCache
          .read()
          .catch((e): Aliases => {
            logger.warn(e, `No module alias info available for ${semester}`);
            return {};
          });
        aliases.push(mapValues(semesterAliases, (moduleCodes) => new Set(moduleCodes)));

        const modules = await new GetSemesterData(semester, year).outputCache.read().catch((e) => {
          logger.warn(e, `No semester data available for ${semester}`);
          return [];
        });
        semesterData.push(modules);
      }

      await new CollateModules().run({ semesterData, aliases });
    }),
  })
  .command({
    command: 'all [year]',
    describe: 'run all tasks in a single pipeline',
    builder: {
      year: parameters.year,
    },
    handler: run(({ year }) => new DataPipeline(year).run()),
  })
  .command({
    command: 'migrate [year]',
    describe: 'move v1 modules to v2 modules format',
    builder: {
      year: parameters.year,
    },
    handler: run(async ({ year }) => {
      // Always use current year because v2 API endpoint may not return data for past years
      const input = await new GetFacultyDepartment(config.academicYear).run();
      const convertedModules = await new MigrateV1ToV2(year).run(input);
      logger.info(`Converted ${convertedModules.length} modules`);
    }),
  })
  .demandCommand()
  .strict()
  .help().argv;