feat(blacklist): Automatically add media with blacklisted tags to the blacklist (#1306)

* feat(blacklist): add blacktag settings to main settings page

* feat(blacklist): create blacktag logic and infrastructure

* feat(blacklist): add scheduling for blacktags job

* feat(blacklist): create blacktag ui badge for blacklist

* docs(blacklist): document blacktags in using-jellyseerr

* fix(blacklist): batch blacklist and media db removes to avoid expression tree too large error

* feat(blacklist): allow easy import and export of blacktag configuration

* fix(settings): don't copy the API key every time you press enter on the main settings

* fix(blacklist): move filter inline with page title to match all the other pages

* feat(blacklist): allow filtering between manually blacklisted and automatically blacklisted entries

* docs(blacklist): reword blacktag documentation a little

* refactor(blacklist): remove blacktag settings from public settings interfaces

There's no reason for it to be there

* refactor(blacklist): remove unused variable from processResults in blacktagsProcessor

* refactor(blacklist): change all instances of blacktag to blacklistedTag and update doc to match

* docs(blacklist): update general documentation for blacklisted tag settings

* fix(blacklist): update setting use of "blacklisted tag" to match between modals

* perf(blacklist): remove media type constraint from existing blacklist entry query

Doesn't make sense to keep it because tmdbid has a unique constraint on it

* fix(blacklist): remove whitespace line causing prettier to fail in CI

* refactor(blacklist): swap out some != and == for !s and _s

* fix(blacklist): merge back CopyButton changes, disable button when there's nothing to copy

* refactor(blacklist): use axios instead of fetch for blacklisted tag queries

* style(blacklist): use templated axios types and remove redundant try-catches

server/api/themoviedb/index.ts

@@ -38,23 +38,26 @@ interface SingleSearchOptions extends SearchOptions {
   year?: number;
 }
 
-export type SortOptions =
-  | 'popularity.asc'
-  | 'popularity.desc'
-  | 'release_date.asc'
-  | 'release_date.desc'
-  | 'revenue.asc'
-  | 'revenue.desc'
-  | 'primary_release_date.asc'
-  | 'primary_release_date.desc'
-  | 'original_title.asc'
-  | 'original_title.desc'
-  | 'vote_average.asc'
-  | 'vote_average.desc'
-  | 'vote_count.asc'
-  | 'vote_count.desc'
-  | 'first_air_date.asc'
-  | 'first_air_date.desc';
+export const SortOptionsIterable = [
+  'popularity.desc',
+  'popularity.asc',
+  'release_date.desc',
+  'release_date.asc',
+  'revenue.desc',
+  'revenue.asc',
+  'primary_release_date.desc',
+  'primary_release_date.asc',
+  'original_title.asc',
+  'original_title.desc',
+  'vote_average.desc',
+  'vote_average.asc',
+  'vote_count.desc',
+  'vote_count.asc',
+  'first_air_date.desc',
+  'first_air_date.asc',
+] as const;
+
+export type SortOptions = (typeof SortOptionsIterable)[number];
 
 interface DiscoverMovieOptions {
   page?: number;

server/entity/Blacklist.ts

@@ -1,8 +1,9 @@
 import { MediaStatus, type MediaType } from '@server/constants/media';
-import { getRepository } from '@server/datasource';
+import dataSource from '@server/datasource';
 import Media from '@server/entity/Media';
 import { User } from '@server/entity/User';
 import type { BlacklistItem } from '@server/interfaces/api/blacklistInterfaces';
+import type { EntityManager } from 'typeorm';
 import {
   Column,
   CreateDateColumn,
@@ -35,7 +36,7 @@ export class Blacklist implements BlacklistItem {
   @ManyToOne(() => User, (user) => user.id, {
     eager: true,
   })
-  user: User;
+  user?: User;
 
   @OneToOne(() => Media, (media) => media.blacklist, {
     onDelete: 'CASCADE',
@@ -43,6 +44,9 @@ export class Blacklist implements BlacklistItem {
   @JoinColumn()
   public media: Media;
 
+  @Column({ nullable: true, type: 'varchar' })
+  public blacklistedTags?: string;
+
   @CreateDateColumn()
   public createdAt: Date;
 
@@ -50,27 +54,32 @@ export class Blacklist implements BlacklistItem {
     Object.assign(this, init);
   }
 
-  public static async addToBlacklist({
-    blacklistRequest,
-  }: {
-    blacklistRequest: {
-      mediaType: MediaType;
-      title?: ZodOptional<ZodString>['_output'];
-      tmdbId: ZodNumber['_output'];
-    };
-  }): Promise<void> {
+  public static async addToBlacklist(
+    {
+      blacklistRequest,
+    }: {
+      blacklistRequest: {
+        mediaType: MediaType;
+        title?: ZodOptional<ZodString>['_output'];
+        tmdbId: ZodNumber['_output'];
+        blacklistedTags?: string;
+      };
+    },
+    entityManager?: EntityManager
+  ): Promise<void> {
+    const em = entityManager ?? dataSource;
     const blacklist = new this({
       ...blacklistRequest,
     });
 
-    const mediaRepository = getRepository(Media);
+    const mediaRepository = em.getRepository(Media);
     let media = await mediaRepository.findOne({
       where: {
         tmdbId: blacklistRequest.tmdbId,
       },
     });
 
-    const blacklistRepository = getRepository(this);
+    const blacklistRepository = em.getRepository(this);
 
     await blacklistRepository.save(blacklist);
 

server/interfaces/api/blacklistInterfaces.ts

@@ -6,7 +6,8 @@ export interface BlacklistItem {
   mediaType: 'movie' | 'tv';
   title?: string;
   createdAt?: Date;
-  user: User;
+  user?: User;
+  blacklistedTags?: string;
 }
 
 export interface BlacklistResultsResponse extends PaginatedResponse {

server/job/blacklistedTagsProcessor.ts

@@ -0,0 +1,184 @@
+import type { SortOptions } from '@server/api/themoviedb';
+import { SortOptionsIterable } from '@server/api/themoviedb';
+import type {
+  TmdbSearchMovieResponse,
+  TmdbSearchTvResponse,
+} from '@server/api/themoviedb/interfaces';
+import { MediaType } from '@server/constants/media';
+import dataSource from '@server/datasource';
+import { Blacklist } from '@server/entity/Blacklist';
+import Media from '@server/entity/Media';
+import type {
+  RunnableScanner,
+  StatusBase,
+} from '@server/lib/scanners/baseScanner';
+import { getSettings } from '@server/lib/settings';
+import logger from '@server/logger';
+import { createTmdbWithRegionLanguage } from '@server/routes/discover';
+import type { EntityManager } from 'typeorm';
+
+const TMDB_API_DELAY_MS = 250;
+class AbortTransaction extends Error {}
+
+class BlacklistedTagProcessor implements RunnableScanner<StatusBase> {
+  private running = false;
+  private progress = 0;
+  private total = 0;
+
+  public async run() {
+    this.running = true;
+
+    try {
+      await dataSource.transaction(async (em) => {
+        await this.cleanBlacklist(em);
+        await this.createBlacklistEntries(em);
+      });
+    } catch (err) {
+      if (err instanceof AbortTransaction) {
+        logger.info('Aborting job: Process Blacklisted Tags', {
+          label: 'Jobs',
+        });
+      } else {
+        throw err;
+      }
+    } finally {
+      this.reset();
+    }
+  }
+
+  public status(): StatusBase {
+    return {
+      running: this.running,
+      progress: this.progress,
+      total: this.total,
+    };
+  }
+
+  public cancel() {
+    this.running = false;
+    this.progress = 0;
+    this.total = 0;
+  }
+
+  private reset() {
+    this.cancel();
+  }
+
+  private async createBlacklistEntries(em: EntityManager) {
+    const tmdb = createTmdbWithRegionLanguage();
+
+    const settings = getSettings();
+    const blacklistedTags = settings.main.blacklistedTags;
+    const blacklistedTagsArr = blacklistedTags.split(',');
+
+    const pageLimit = settings.main.blacklistedTagsLimit;
+
+    if (blacklistedTags.length === 0) {
+      return;
+    }
+
+    // The maximum number of queries we're expected to execute
+    this.total =
+      2 * blacklistedTagsArr.length * pageLimit * SortOptionsIterable.length;
+
+    for (const type of [MediaType.MOVIE, MediaType.TV]) {
+      const getDiscover =
+        type === MediaType.MOVIE ? tmdb.getDiscoverMovies : tmdb.getDiscoverTv;
+
+      // Iterate for each tag
+      for (const tag of blacklistedTagsArr) {
+        let queryMax = pageLimit * SortOptionsIterable.length;
+        let fixedSortMode = false; // Set to true when the page limit allows for getting every page of tag
+
+        for (let query = 0; query < queryMax; query++) {
+          const page: number = fixedSortMode
+            ? query + 1
+            : (query % pageLimit) + 1;
+          const sortBy: SortOptions | undefined = fixedSortMode
+            ? undefined
+            : SortOptionsIterable[query % SortOptionsIterable.length];
+
+          if (!this.running) {
+            throw new AbortTransaction();
+          }
+
+          const response = await getDiscover({
+            page,
+            sortBy,
+            keywords: tag,
+          });
+          await this.processResults(response, tag, type, em);
+          await new Promise((res) => setTimeout(res, TMDB_API_DELAY_MS));
+
+          this.progress++;
+          if (page === 1 && response.total_pages <= queryMax) {
+            // We will finish the tag with less queries than expected, move progress accordingly
+            this.progress += queryMax - response.total_pages;
+            fixedSortMode = true;
+            queryMax = response.total_pages;
+          }
+        }
+      }
+    }
+  }
+
+  private async processResults(
+    response: TmdbSearchMovieResponse | TmdbSearchTvResponse,
+    keywordId: string,
+    mediaType: MediaType,
+    em: EntityManager
+  ) {
+    const blacklistRepository = em.getRepository(Blacklist);
+
+    for (const entry of response.results) {
+      const blacklistEntry = await blacklistRepository.findOne({
+        where: { tmdbId: entry.id },
+      });
+
+      if (blacklistEntry) {
+        // Don't mark manual blacklists with tags
+        // If media wasn't previously blacklisted for this tag, add the tag to the media's blacklist
+        if (
+          blacklistEntry.blacklistedTags &&
+          !blacklistEntry.blacklistedTags.includes(`,${keywordId},`)
+        ) {
+          await blacklistRepository.update(blacklistEntry.id, {
+            blacklistedTags: `${blacklistEntry.blacklistedTags}${keywordId},`,
+          });
+        }
+      } else {
+        // Media wasn't previously blacklisted, add it to the blacklist
+        await Blacklist.addToBlacklist(
+          {
+            blacklistRequest: {
+              mediaType,
+              title: 'title' in entry ? entry.title : entry.name,
+              tmdbId: entry.id,
+              blacklistedTags: `,${keywordId},`,
+            },
+          },
+          em
+        );
+      }
+    }
+  }
+
+  private async cleanBlacklist(em: EntityManager) {
+    // Remove blacklist and media entries blacklisted by tags
+    const mediaRepository = em.getRepository(Media);
+    const mediaToRemove = await mediaRepository
+      .createQueryBuilder('media')
+      .innerJoinAndSelect(Blacklist, 'blist', 'blist.tmdbId = media.tmdbId')
+      .where(`blist.blacklistedTags IS NOT NULL`)
+      .getMany();
+
+    // Batch removes so the query doesn't get too large
+    for (let i = 0; i < mediaToRemove.length; i += 500) {
+      await mediaRepository.remove(mediaToRemove.slice(i, i + 500)); // This also deletes the blacklist entries via cascading
+    }
+  }
+}
+
+const blacklistedTagsProcessor = new BlacklistedTagProcessor();
+
+export default blacklistedTagsProcessor;

server/job/schedule.ts

@@ -1,4 +1,5 @@
 import { MediaServerType } from '@server/constants/server';
+import blacklistedTagsProcessor from '@server/job/blacklistedTagsProcessor';
 import availabilitySync from '@server/lib/availabilitySync';
 import downloadTracker from '@server/lib/downloadtracker';
 import ImageProxy from '@server/lib/imageproxy';
@@ -21,7 +22,7 @@ interface ScheduledJob {
   job: schedule.Job;
   name: string;
   type: 'process' | 'command';
-  interval: 'seconds' | 'minutes' | 'hours' | 'fixed';
+  interval: 'seconds' | 'minutes' | 'hours' | 'days' | 'fixed';
   cronSchedule: string;
   running?: () => boolean;
   cancelFn?: () => void;
@@ -237,5 +238,21 @@ export const startJobs = (): void => {
     }),
   });
 
+  scheduledJobs.push({
+    id: 'process-blacklisted-tags',
+    name: 'Process Blacklisted Tags',
+    type: 'process',
+    interval: 'days',
+    cronSchedule: jobs['process-blacklisted-tags'].schedule,
+    job: schedule.scheduleJob(jobs['process-blacklisted-tags'].schedule, () => {
+      logger.info('Starting scheduled job: Process Blacklisted Tags', {
+        label: 'Jobs',
+      });
+      blacklistedTagsProcessor.run();
+    }),
+    running: () => blacklistedTagsProcessor.status().running,
+    cancelFn: () => blacklistedTagsProcessor.cancel(),
+  });
+
   logger.info('Scheduled jobs loaded', { label: 'Jobs' });
 };

server/lib/settings/index.ts

@@ -128,6 +128,8 @@ export interface MainSettings {
   discoverRegion: string;
   streamingRegion: string;
   originalLanguage: string;
+  blacklistedTags: string;
+  blacklistedTagsLimit: number;
   mediaServerType: number;
   partialRequestsEnabled: boolean;
   enableSpecialEpisodes: boolean;
@@ -302,7 +304,8 @@ export type JobId =
   | 'jellyfin-recently-added-scan'
   | 'jellyfin-full-scan'
   | 'image-cache-cleanup'
-  | 'availability-sync';
+  | 'availability-sync'
+  | 'process-blacklisted-tags';
 
 export interface AllSettings {
   clientId: string;
@@ -349,6 +352,8 @@ class Settings {
         discoverRegion: '',
         streamingRegion: '',
         originalLanguage: '',
+        blacklistedTags: '',
+        blacklistedTagsLimit: 50,
         mediaServerType: MediaServerType.NOT_CONFIGURED,
         partialRequestsEnabled: true,
         enableSpecialEpisodes: false,
@@ -505,6 +510,9 @@ class Settings {
         'image-cache-cleanup': {
           schedule: '0 0 5 * * *',
         },
+        'process-blacklisted-tags': {
+          schedule: '0 30 1 */7 * *',
+        },
       },
       network: {
         csrfProtection: false,

server/migration/postgres/1737320080282-AddBlacklistTagsColumn.ts

@@ -0,0 +1,17 @@
+import type { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddBlacklistTagsColumn1737320080282 implements MigrationInterface {
+  name = 'AddBlacklistTagsColumn1737320080282';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "blacklist" ADD blacklistedTags character varying`
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "blacklist" DROP COLUMN blacklistedTags`
+    );
+  }
+}

server/migration/sqlite/1737320080282-AddBlacklistTagsColumn.ts

@@ -0,0 +1,34 @@
+import type { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddBlacklistTagsColumn1737320080282 implements MigrationInterface {
+  name = 'AddBlacklistTagsColumn1737320080282';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "temporary_blacklist" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "mediaType" varchar NOT NULL, "title" varchar, "tmdbId" integer NOT NULL, "blacklistedTags" varchar, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "userId" integer, "mediaId" integer, CONSTRAINT "UQ_6bbafa28411e6046421991ea21c" UNIQUE ("tmdbId"), CONSTRAINT "REL_62b7ade94540f9f8d8bede54b9" UNIQUE ("mediaId"), CONSTRAINT "FK_53c1ab62c3e5875bc3ac474823e" FOREIGN KEY ("userId") REFERENCES "user" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, CONSTRAINT "FK_62b7ade94540f9f8d8bede54b99" FOREIGN KEY ("mediaId") REFERENCES "media" ("id") ON DELETE CASCADE ON UPDATE NO ACTION)`
+    );
+    await queryRunner.query(
+      `INSERT INTO "temporary_blacklist"("id", "mediaType", "title", "tmdbId", "blacklistedTags", "createdAt", "userId", "mediaId") SELECT "id", "mediaType", "title", "tmdbId", "blacklistedTags", "createdAt", "userId", "mediaId" FROM "blacklist"`
+    );
+    await queryRunner.query(`DROP TABLE "blacklist"`);
+    await queryRunner.query(
+      `ALTER TABLE "temporary_blacklist" RENAME TO "blacklist"`
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_6bbafa28411e6046421991ea21" ON "blacklist" ("tmdbId") `
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "blacklist" RENAME TO "temporary_blacklist"`
+    );
+    await queryRunner.query(
+      `CREATE TABLE "blacklist" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "mediaType" varchar NOT NULL, "title" varchar, "tmdbId" integer NOT NULL, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "userId" integer, "mediaId" integer, CONSTRAINT "UQ_6bbafa28411e6046421991ea21c" UNIQUE ("tmdbId"), CONSTRAINT "REL_62b7ade94540f9f8d8bede54b9" UNIQUE ("mediaId"))`
+    );
+    await queryRunner.query(
+      `INSERT INTO "blacklist"("id", "mediaType", "title", "tmdbId", "createdAt", "userId", "mediaId") SELECT "id", "mediaType", "title", "tmdbId", "createdAt", "userId", "mediaId" FROM "temporary_blacklist"`
+    );
+    await queryRunner.query(`DROP TABLE "temporary_blacklist"`);
+  }
+}

server/routes/blacklist.ts

@@ -19,39 +19,54 @@ export const blacklistAdd = z.object({
   user: z.coerce.number(),
 });
 
+const blacklistGet = z.object({
+  take: z.coerce.number().int().positive().default(25),
+  skip: z.coerce.number().int().nonnegative().default(0),
+  search: z.string().optional(),
+  filter: z.enum(['all', 'manual', 'blacklistedTags']).optional(),
+});
+
 blacklistRoutes.get(
   '/',
   isAuthenticated([Permission.MANAGE_BLACKLIST, Permission.VIEW_BLACKLIST], {
     type: 'or',
   }),
   async (req, res, next) => {
-    const pageSize = req.query.take ? Number(req.query.take) : 25;
-    const skip = req.query.skip ? Number(req.query.skip) : 0;
-    const search = (req.query.search as string) ?? '';
+    const { take, skip, search, filter } = blacklistGet.parse(req.query);
 
     try {
       let query = getRepository(Blacklist)
         .createQueryBuilder('blacklist')
-        .leftJoinAndSelect('blacklist.user', 'user');
+        .leftJoinAndSelect('blacklist.user', 'user')
+        .where('1 = 1'); // Allow use of andWhere later
 
-      if (search.length > 0) {
-        query = query.where('blacklist.title like :title', {
+      switch (filter) {
+        case 'manual':
+          query = query.andWhere('blacklist.blacklistedTags IS NULL');
+          break;
+        case 'blacklistedTags':
+          query = query.andWhere('blacklist.blacklistedTags IS NOT NULL');
+          break;
+      }
+
+      if (search) {
+        query = query.andWhere('blacklist.title like :title', {
           title: `%${search}%`,
         });
       }
 
       const [blacklistedItems, itemsCount] = await query
         .orderBy('blacklist.createdAt', 'DESC')
-        .take(pageSize)
+        .take(take)
         .skip(skip)
         .getManyAndCount();
 
       return res.status(200).json({
         pageInfo: {
-          pages: Math.ceil(itemsCount / pageSize),
-          pageSize,
+          pages: Math.ceil(itemsCount / take),
+          pageSize: take,
           results: itemsCount,
-          page: Math.ceil(skip / pageSize) + 1,
+          page: Math.ceil(skip / take) + 1,
         },
         results: blacklistedItems,
       } as BlacklistResultsResponse);