fix: reduce duplicate image downloads

build/build.ts

@@ -25,6 +25,8 @@ import type {
   CategoryData,
   ApiCategory,
 } from './types/shared';
+import { existsSync } from 'node:fs';
+import sanitize from 'sanitize-filename';
 
 let imageCache: CachedItem[] = [];
 
@@ -63,9 +65,10 @@ class Build {
     const parsed = parser.parse(raw);
     const data = this.applyCustomCategories(parsed.data);
     const i18n = parser.applyI18n(data, raw.i18n);
-    const all = await this.saveJson(data, i18n);
+    const all = this.merge(data);
     await this.saveWarnings(parsed.warnings);
     await this.saveImages(all, raw.manifest);
+    await this.saveJson(all, data, i18n);
     await this.updateReadme(raw.patchlogs);
 
     // Log number of warnings at the end of the script
@@ -114,44 +117,55 @@ class Build {
     return result;
   }
 
+  sort(a: Item, b: Item): number {
+    if (!a.name) console.log(a);
+    const res = a.name.localeCompare(b.name);
+    if (res === 0) {
+      return a.uniqueName.localeCompare(b.uniqueName);
+    }
+    return res;
+  }
+
+  merge(categories: Record<string, Item[]>): Item[] {
+    let all: Item[] = [];
+
+    // Category names are provided by this.applyCustomCategories
+    for (const category of Object.keys(categories)) {
+      const categoryData = categories[category];
+      if (!categoryData) continue;
+      all = all.concat(categoryData);
+    }
+
+    // All.json (all items in one file)
+    all.sort(this.sort.bind(this));
+
+    return all;
+  }
+
   /**
    * Generate JSON file for each category and one for all combined.
    * @param categories list of categories to save and separate
    * @param i18n internationalization partials of Items
    * @returns all items
    */
   async saveJson(
+    bundle: Item[],
     categories: Record<string, Item[]>,
     i18n: Record<string, Record<string, Partial<Item>>>
-  ): Promise<Item[]> {
-    let all: Item[] = [];
-    const sort = (a: Item, b: Item): number => {
-      if (!a.name) console.log(a);
-      const res = a.name.localeCompare(b.name);
-      if (res === 0) {
-        return a.uniqueName.localeCompare(b.uniqueName);
-      }
-      return res;
-    };
-
+  ): Promise<void> {
     // Category names are provided by this.applyCustomCategories
     for (const category of Object.keys(categories)) {
       const categoryData = categories[category];
       if (!categoryData) continue;
-      const data = categoryData.sort(sort);
-      all = all.concat(data);
+      const data = categoryData.sort(this.sort.bind(this));
       await fs.writeFile(
         new URL(`../data/json/${category}.json`, import.meta.url),
         JSON.stringify(JSON.parse(stringify(data)))
       );
     }
 
-    // All.json (all items in one file)
-    all.sort(sort);
-    await fs.writeFile(new URL('../data/json/All.json', import.meta.url), stringify(all));
+    await fs.writeFile(new URL('../data/json/All.json', import.meta.url), stringify(bundle));
     await fs.writeFile(new URL('../data/json/i18n.json', import.meta.url), JSON.stringify(JSON.parse(stringify(i18n))));
-
-    return all;
   }
 
   /**
@@ -173,21 +187,21 @@ class Build {
     // hash, so any change to that changes the hash of the full thing.
     if (!hashManager.hasChanged('Manifest')) return;
     const bar = new Progress('Fetching Images', items.length);
-    const duplicates: string[] = []; // Don't download component images or relics twice
+    const processed: Record<string, string> = {}; // Don't download component images or relics twice
 
     for (const item of items) {
       // Save image for parent item
-      await this.saveImage(item, false, duplicates, manifest);
+      await this.saveImage(item, false, processed, manifest);
       // Save images for components if necessary
       if (item.components) {
         for (const component of item.components) {
-          await this.saveImage(component, true, duplicates, manifest);
+          await this.saveImage(component, true, processed, manifest);
         }
       }
       // Save images for abilities
       if (item.abilities) {
         for (const ability of item.abilities) {
-          await this.saveImage(ability as Item, false, duplicates, manifest);
+          await this.saveImage(ability as Item, false, processed, manifest);
         }
       }
       bar.tick();
@@ -208,41 +222,48 @@ class Build {
    * Download and save images for items or components.
    * @param item to determine and save an image for
    * @param isComponent whether the item is a component or a parent
-   * @param duplicates list of duplicated (already existing) image names
+   * @param processed list of duplicated (already existing) image names
    * @param manifest image lookup list
    */
-  async saveImage(item: Item, isComponent: boolean, duplicates: string[], manifest: ImageManifest): Promise<void> {
-    let { uniqueName } = item;
-    if (item.type === 'Nightwave Act') {
-      uniqueName = item.uniqueName.replace(/[0-9]{1,3}$/, '');
-    }
-
-    const imageBase = manifest.find((i) => i.uniqueName === uniqueName);
+  async saveImage(
+    item: Item,
+    isComponent: boolean,
+    processed: Record<string, string>,
+    manifest: ImageManifest
+  ): Promise<void> {
+    const imageBase = manifest.find((i) => i.uniqueName === item.uniqueName);
     if (!imageBase) return;
 
     const imageStub = imageBase.textureLocation.replace(/\\/g, '/').replace('xport/', '');
     const imageHash = /!00_([\S]+)/.exec(imageStub);
     const imageUrl = `https://content.warframe.com/PublicExport/${imageStub}`;
     const basePath = fileURLToPath(new URL('../data/img/', import.meta.url));
-    const filePath = path.join(basePath, item.imageName);
-    const manifestItem = manifest.find((i) => i.uniqueName === item.uniqueName);
-    const hash = manifestItem?.fileTime ?? imageHash?.[1] ?? undefined;
+    let filePath = path.join(basePath, item.imageName);
+    const hash = imageBase?.fileTime ?? imageHash?.[1] ?? undefined;
     const cached = imageCache.find((c) => c.uniqueName === item.uniqueName);
 
     // We'll use a custom blueprint image
     if (item.name === 'Blueprint' || item.name === 'Arcane') return;
 
-    // Don't download component images or relic images twice
-    if (isComponent || item.type === 'Relic') {
-      if (duplicates.includes(item.imageName)) {
-        return;
-      }
-      duplicates.push(item.imageName);
+    // Don't download texture images twice
+    const imageName = processed[hash];
+    if (imageName !== undefined) {
+      item.imageName = imageName;
+      return;
+    }
+
+    processed[hash] = item.imageName;
+
+    // Check for an already exisitng file and fall back to item name
+    if (existsSync(filePath)) {
+      const [_, ext] = item.imageName.split('.');
+      /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
+      item.imageName = sanitize(`${item.name.replace(/[ /*]/g, '-')}.${ext!}`);
+      processed[hash] = item.imageName;
+      filePath = path.join(basePath, item.imageName);
     }
 
-    // Check if the previous image was for a component because they might
-    // have different naming schemes like lex-prime
-    if (!cached || cached.hash !== hash || cached.isComponent !== isComponent) {
+    if (cached?.hash !== hash) {
       try {
         const retry = (err: Error & { code?: string }): Promise<Buffer | string> => {
           if (err.code === 'ENOTFOUND') {

build/parser.ts

@@ -1,5 +1,3 @@
-import { createHash } from 'node:crypto';
-
 import cloneDeep from 'lodash.clonedeep';
 import sanitize from 'sanitize-filename';
 
@@ -567,79 +565,27 @@ class Parser {
       return;
     }
 
-    const encode = (str: string): string =>
-      sanitize(
-        str
-          .replace('/', '')
-          .replace(/[ /*]/g, '-')
-          .replace(/[:<>[\]?!"]/g, '')
-          .toLowerCase()
-      );
-
-    const imageStub = image.textureLocation;
-    const parts = imageStub.split('.');
-    const ext = (parts[parts.length - 1] ?? 'png').replace(/\?!.*/, '').replace(/!.*$/, ''); // .png, .jpg, etc
-    const hash = (str: string): string => createHash('sha256').update(str).digest('hex');
-
     // Enforce arcane and blueprint image name
-    if (item.name === 'Arcane') {
-      item.imageName = `arcane.${ext}`;
+    if (item.name === 'Arcane' || item.name === 'Blueprint') {
+      if (item.name === 'Arcane') item.imageName = `arcane.png`;
+      if (item.name === 'Blueprint') item.imageName = `blueprint.png`;
       return;
     }
-    if (item.name === 'Blueprint') {
-      item.imageName = `blueprint.${ext}`;
-      return;
-    }
-
-    // Turn any separators into dashes and remove characters that would break
-    // the filesystem.
-    item.imageName = encode(item.name);
-
-    if (item.type === 'Nightwave Challenge') {
-      const name = item.name.replace(/\sM{0,3}(CM|CD|D?C{0,3})?(XC|XL|L?X{0,3})?(IX|IV|V?I{0,3})?$/i, '').trim();
-      const sterilized = item.uniqueName.replace(/[0-9]{1,3}$/, '');
-      item.imageName = `${encode(name)}-${hash(sterilized).slice(0, 10)}.${ext}`;
 
+    const imageStub = image.textureLocation;
+    const uniqueName = imageStub.split('!')[0];
+    if (uniqueName === undefined) {
+      warnings.missingImage.push(item.name);
       return;
     }
 
-    // Components usually have the same generic images, so we should remove the
-    // parent name here. Note that there's a difference between prime/non-prime
-    // components, so we'll keep the prime in the name.
-    if (item.parent) {
-      item.imageName = item.imageName.replace(`${encode(item.parent)}-`, '');
-      if (item.name.includes('Prime')) {
-        item.imageName = `prime-${item.imageName}`;
-        // check if the image name ends with prime as some older prime secondaries use the full parent name
-        if (item.imageName.endsWith('prime')) item.imageName = item.imageName.replace(/-prime$/, '');
-      }
-    }
-
-    // Relics should use the same image based on type, as they all use the same.
-    // The resulting format looks like `axi-intact`, `axi-radiant`
-    if (item.type === 'Relic') {
-      item.imageName = item.imageName.replace(/-(.*?)-/, '-'); // Remove second word (type)
-    }
-
-    // Remove the mark number and house name from Railjack weapons
-    if (item.productCategory === 'CrewShipWeapons') {
-      item.imageName = item.imageName.replace(/(lavan|vidar|zetki)-|(-mk-i+)/g, '');
-
-      // Add original file extension
-      item.imageName += `.${ext}`;
+    const imageName = uniqueName.split('/').reverse()[0];
+    if (imageName === undefined) {
+      warnings.missingImage.push(item.name);
       return;
     }
 
-    // Some items have the same name - so add a partial hash as an identifier
-    // but avoid making component images different
-    //
-    // Regex avoids Warframe componenets and Necramech weapons and suit
-    if (item.type !== 'Relic' && !/Recipes|(Resources\/Mechs)/.test(item.uniqueName)) {
-      item.imageName += `-${hash(item.uniqueName).slice(0, 10)}`;
-    }
-
-    // Add original file extension
-    item.imageName += `.${ext}`;
+    item.imageName = sanitize(imageName);
   }
 
   /**

test/utilities/find.spec.mjs

@@ -58,7 +58,7 @@ describe('#loadMods', () => {
       uniqueName: '/Lotus/Upgrades/Mods/Randomized/LotusRifleRandomModRare',
       polarity: 'Vazarin',
       rarity: 'Common',
-      imageName: 'rifle-riven-mod-e05c5519f1.png',
+      imageName: 'OmegaMod.png',
       category: 'Mods',
       buffs: [
         { tag: 'WeaponCritDamageMod', val: 0.3296302411049886 },