feat(blacklist): Automatically add media with blacklisted tags to the blacklist (#1306)

* feat(blacklist): add blacktag settings to main settings page

* feat(blacklist): create blacktag logic and infrastructure

* feat(blacklist): add scheduling for blacktags job

* feat(blacklist): create blacktag ui badge for blacklist

* docs(blacklist): document blacktags in using-jellyseerr

* fix(blacklist): batch blacklist and media db removes to avoid expression tree too large error

* feat(blacklist): allow easy import and export of blacktag configuration

* fix(settings): don't copy the API key every time you press enter on the main settings

* fix(blacklist): move filter inline with page title to match all the other pages

* feat(blacklist): allow filtering between manually blacklisted and automatically blacklisted entries

* docs(blacklist): reword blacktag documentation a little

* refactor(blacklist): remove blacktag settings from public settings interfaces

There's no reason for it to be there

* refactor(blacklist): remove unused variable from processResults in blacktagsProcessor

* refactor(blacklist): change all instances of blacktag to blacklistedTag and update doc to match

* docs(blacklist): update general documentation for blacklisted tag settings

* fix(blacklist): update setting use of "blacklisted tag" to match between modals

* perf(blacklist): remove media type constraint from existing blacklist entry query

Doesn't make sense to keep it because tmdbid has a unique constraint on it

* fix(blacklist): remove whitespace line causing prettier to fail in CI

* refactor(blacklist): swap out some != and == for !s and _s

* fix(blacklist): merge back CopyButton changes, disable button when there's nothing to copy

* refactor(blacklist): use axios instead of fetch for blacklisted tag queries

* style(blacklist): use templated axios types and remove redundant try-catches

server/job/blacklistedTagsProcessor.ts

@@ -0,0 +1,184 @@
+import type { SortOptions } from '@server/api/themoviedb';
+import { SortOptionsIterable } from '@server/api/themoviedb';
+import type {
+  TmdbSearchMovieResponse,
+  TmdbSearchTvResponse,
+} from '@server/api/themoviedb/interfaces';
+import { MediaType } from '@server/constants/media';
+import dataSource from '@server/datasource';
+import { Blacklist } from '@server/entity/Blacklist';
+import Media from '@server/entity/Media';
+import type {
+  RunnableScanner,
+  StatusBase,
+} from '@server/lib/scanners/baseScanner';
+import { getSettings } from '@server/lib/settings';
+import logger from '@server/logger';
+import { createTmdbWithRegionLanguage } from '@server/routes/discover';
+import type { EntityManager } from 'typeorm';
+
+const TMDB_API_DELAY_MS = 250;
+class AbortTransaction extends Error {}
+
+class BlacklistedTagProcessor implements RunnableScanner<StatusBase> {
+  private running = false;
+  private progress = 0;
+  private total = 0;
+
+  public async run() {
+    this.running = true;
+
+    try {
+      await dataSource.transaction(async (em) => {
+        await this.cleanBlacklist(em);
+        await this.createBlacklistEntries(em);
+      });
+    } catch (err) {
+      if (err instanceof AbortTransaction) {
+        logger.info('Aborting job: Process Blacklisted Tags', {
+          label: 'Jobs',
+        });
+      } else {
+        throw err;
+      }
+    } finally {
+      this.reset();
+    }
+  }
+
+  public status(): StatusBase {
+    return {
+      running: this.running,
+      progress: this.progress,
+      total: this.total,
+    };
+  }
+
+  public cancel() {
+    this.running = false;
+    this.progress = 0;
+    this.total = 0;
+  }
+
+  private reset() {
+    this.cancel();
+  }
+
+  private async createBlacklistEntries(em: EntityManager) {
+    const tmdb = createTmdbWithRegionLanguage();
+
+    const settings = getSettings();
+    const blacklistedTags = settings.main.blacklistedTags;
+    const blacklistedTagsArr = blacklistedTags.split(',');
+
+    const pageLimit = settings.main.blacklistedTagsLimit;
+
+    if (blacklistedTags.length === 0) {
+      return;
+    }
+
+    // The maximum number of queries we're expected to execute
+    this.total =
+      2 * blacklistedTagsArr.length * pageLimit * SortOptionsIterable.length;
+
+    for (const type of [MediaType.MOVIE, MediaType.TV]) {
+      const getDiscover =
+        type === MediaType.MOVIE ? tmdb.getDiscoverMovies : tmdb.getDiscoverTv;
+
+      // Iterate for each tag
+      for (const tag of blacklistedTagsArr) {
+        let queryMax = pageLimit * SortOptionsIterable.length;
+        let fixedSortMode = false; // Set to true when the page limit allows for getting every page of tag
+
+        for (let query = 0; query < queryMax; query++) {
+          const page: number = fixedSortMode
+            ? query + 1
+            : (query % pageLimit) + 1;
+          const sortBy: SortOptions | undefined = fixedSortMode
+            ? undefined
+            : SortOptionsIterable[query % SortOptionsIterable.length];
+
+          if (!this.running) {
+            throw new AbortTransaction();
+          }
+
+          const response = await getDiscover({
+            page,
+            sortBy,
+            keywords: tag,
+          });
+          await this.processResults(response, tag, type, em);
+          await new Promise((res) => setTimeout(res, TMDB_API_DELAY_MS));
+
+          this.progress++;
+          if (page === 1 && response.total_pages <= queryMax) {
+            // We will finish the tag with less queries than expected, move progress accordingly
+            this.progress += queryMax - response.total_pages;
+            fixedSortMode = true;
+            queryMax = response.total_pages;
+          }
+        }
+      }
+    }
+  }
+
+  private async processResults(
+    response: TmdbSearchMovieResponse | TmdbSearchTvResponse,
+    keywordId: string,
+    mediaType: MediaType,
+    em: EntityManager
+  ) {
+    const blacklistRepository = em.getRepository(Blacklist);
+
+    for (const entry of response.results) {
+      const blacklistEntry = await blacklistRepository.findOne({
+        where: { tmdbId: entry.id },
+      });
+
+      if (blacklistEntry) {
+        // Don't mark manual blacklists with tags
+        // If media wasn't previously blacklisted for this tag, add the tag to the media's blacklist
+        if (
+          blacklistEntry.blacklistedTags &&
+          !blacklistEntry.blacklistedTags.includes(`,${keywordId},`)
+        ) {
+          await blacklistRepository.update(blacklistEntry.id, {
+            blacklistedTags: `${blacklistEntry.blacklistedTags}${keywordId},`,
+          });
+        }
+      } else {
+        // Media wasn't previously blacklisted, add it to the blacklist
+        await Blacklist.addToBlacklist(
+          {
+            blacklistRequest: {
+              mediaType,
+              title: 'title' in entry ? entry.title : entry.name,
+              tmdbId: entry.id,
+              blacklistedTags: `,${keywordId},`,
+            },
+          },
+          em
+        );
+      }
+    }
+  }
+
+  private async cleanBlacklist(em: EntityManager) {
+    // Remove blacklist and media entries blacklisted by tags
+    const mediaRepository = em.getRepository(Media);
+    const mediaToRemove = await mediaRepository
+      .createQueryBuilder('media')
+      .innerJoinAndSelect(Blacklist, 'blist', 'blist.tmdbId = media.tmdbId')
+      .where(`blist.blacklistedTags IS NOT NULL`)
+      .getMany();
+
+    // Batch removes so the query doesn't get too large
+    for (let i = 0; i < mediaToRemove.length; i += 500) {
+      await mediaRepository.remove(mediaToRemove.slice(i, i + 500)); // This also deletes the blacklist entries via cascading
+    }
+  }
+}
+
+const blacklistedTagsProcessor = new BlacklistedTagProcessor();
+
+export default blacklistedTagsProcessor;

server/job/schedule.ts

@@ -1,4 +1,5 @@
 import { MediaServerType } from '@server/constants/server';
+import blacklistedTagsProcessor from '@server/job/blacklistedTagsProcessor';
 import availabilitySync from '@server/lib/availabilitySync';
 import downloadTracker from '@server/lib/downloadtracker';
 import ImageProxy from '@server/lib/imageproxy';
@@ -21,7 +22,7 @@ interface ScheduledJob {
   job: schedule.Job;
   name: string;
   type: 'process' | 'command';
-  interval: 'seconds' | 'minutes' | 'hours' | 'fixed';
+  interval: 'seconds' | 'minutes' | 'hours' | 'days' | 'fixed';
   cronSchedule: string;
   running?: () => boolean;
   cancelFn?: () => void;
@@ -237,5 +238,21 @@ export const startJobs = (): void => {
     }),
   });
 
+  scheduledJobs.push({
+    id: 'process-blacklisted-tags',
+    name: 'Process Blacklisted Tags',
+    type: 'process',
+    interval: 'days',
+    cronSchedule: jobs['process-blacklisted-tags'].schedule,
+    job: schedule.scheduleJob(jobs['process-blacklisted-tags'].schedule, () => {
+      logger.info('Starting scheduled job: Process Blacklisted Tags', {
+        label: 'Jobs',
+      });
+      blacklistedTagsProcessor.run();
+    }),
+    running: () => blacklistedTagsProcessor.status().running,
+    cancelFn: () => blacklistedTagsProcessor.cancel(),
+  });
+
   logger.info('Scheduled jobs loaded', { label: 'Jobs' });
 };