fix: reduce duplicate image downloads

build/build.ts

@@ -25,6 +25,9 @@ import type {
   CategoryData,
   ApiCategory,
 } from './types/shared';
+import { existsSync } from 'node:fs';
+import sanitize from 'sanitize-filename';
+import { createHash } from 'node:crypto';
 
 let imageCache: CachedItem[] = [];
 
@@ -63,9 +66,9 @@ class Build {
     const parsed = parser.parse(raw);
     const data = this.applyCustomCategories(parsed.data);
     const i18n = parser.applyI18n(data, raw.i18n);
-    const all = await this.saveJson(data, i18n);
+    await this.saveImages(data, raw.manifest, parsed.warnings);
+    await this.saveJson(data, i18n);
     await this.saveWarnings(parsed.warnings);
-    await this.saveImages(all, raw.manifest);
     await this.updateReadme(raw.patchlogs);
 
     // Log number of warnings at the end of the script
@@ -123,7 +126,7 @@ class Build {
   async saveJson(
     categories: Record<string, Item[]>,
     i18n: Record<string, Record<string, Partial<Item>>>
-  ): Promise<Item[]> {
+  ): Promise<void> {
     let all: Item[] = [];
     const sort = (a: Item, b: Item): number => {
       if (!a.name) console.log(a);
@@ -150,8 +153,6 @@ class Build {
     all.sort(sort);
     await fs.writeFile(new URL('../data/json/All.json', import.meta.url), stringify(all));
     await fs.writeFile(new URL('../data/json/i18n.json', import.meta.url), JSON.stringify(JSON.parse(stringify(i18n))));
-
-    return all;
   }
 
   /**
@@ -167,30 +168,40 @@ class Build {
    * @param items items to append images to
    * @param manifest image manifest to look up items from
    */
-  async saveImages(items: Item[], manifest: ImageManifest): Promise<void> {
+  async saveImages(categories: Record<string, Item[]>, manifest: ImageManifest, warnings: Warnings): Promise<void> {
     // No need to go through every item if the manifest didn't change. I'm
     // guessing the `fileTime` key in each element works more or less like a
     // hash, so any change to that changes the hash of the full thing.
-    if (!hashManager.hasChanged('Manifest')) return;
+    // if (!hashManager.hasChanged('Manifest')) return;
+    const items = Object.values(categories).flat();
     const bar = new Progress('Fetching Images', items.length);
-    const duplicates: string[] = []; // Don't download component images or relics twice
-
-    for (const item of items) {
-      // Save image for parent item
-      await this.saveImage(item, false, duplicates, manifest);
-      // Save images for components if necessary
-      if (item.components) {
-        for (const component of item.components) {
-          await this.saveImage(component, true, duplicates, manifest);
-        }
-      }
-      // Save images for abilities
-      if (item.abilities) {
-        for (const ability of item.abilities) {
-          await this.saveImage(ability as Item, false, duplicates, manifest);
+    const processed: Record<string, string> = {}; // Don't download component images or relics twice
+
+    for (const category of Object.keys(categories)) {
+      const categoryData = categories[category];
+      if (!categoryData) continue;
+
+      for (const item of categoryData) {
+        try {
+          // Save image for parent item
+          await this.saveImage(item, false, processed, manifest);
+          // Save images for components if necessary
+          if (item.components) {
+            for (const component of item.components) {
+              await this.saveImage(component, true, processed, manifest);
+            }
+          }
+          // Save images for abilities
+          if (item.abilities) {
+            for (const ability of item.abilities) {
+              await this.saveImage(ability as Item, false, processed, manifest);
+            }
+          }
+        } catch {
+          warnings.missingImage.push(item.name);
         }
+        bar.tick();
       }
-      bar.tick();
     }
 
     // write the manifests after images have all succeeded
@@ -208,41 +219,49 @@ class Build {
    * Download and save images for items or components.
    * @param item to determine and save an image for
    * @param isComponent whether the item is a component or a parent
-   * @param duplicates list of duplicated (already existing) image names
+   * @param processed list of duplicated (already existing) image names
    * @param manifest image lookup list
    */
-  async saveImage(item: Item, isComponent: boolean, duplicates: string[], manifest: ImageManifest): Promise<void> {
-    let { uniqueName } = item;
-    if (item.type === 'Nightwave Act') {
-      uniqueName = item.uniqueName.replace(/[0-9]{1,3}$/, '');
-    }
-
-    const imageBase = manifest.find((i) => i.uniqueName === uniqueName);
+  async saveImage(
+    item: Item,
+    isComponent: boolean,
+    processed: Record<string, string>,
+    manifest: ImageManifest
+  ): Promise<void> {
+    const imageBase = manifest.find((i) => i.uniqueName === item.uniqueName);
     if (!imageBase) return;
 
     const imageStub = imageBase.textureLocation.replace(/\\/g, '/').replace('xport/', '');
     const imageHash = /!00_([\S]+)/.exec(imageStub);
     const imageUrl = `https://content.warframe.com/PublicExport/${imageStub}`;
     const basePath = fileURLToPath(new URL('../data/img/', import.meta.url));
-    const filePath = path.join(basePath, item.imageName);
-    const manifestItem = manifest.find((i) => i.uniqueName === item.uniqueName);
-    const hash = manifestItem?.fileTime ?? imageHash?.[1] ?? undefined;
+    let filePath = path.join(basePath, item.imageName);
+    const hash =
+      imageBase?.fileTime ?? imageHash?.[1] ?? createHash('md5').update(imageBase.textureLocation).digest('hex');
     const cached = imageCache.find((c) => c.uniqueName === item.uniqueName);
 
     // We'll use a custom blueprint image
     if (item.name === 'Blueprint' || item.name === 'Arcane') return;
 
-    // Don't download component images or relic images twice
-    if (isComponent || item.type === 'Relic') {
-      if (duplicates.includes(item.imageName)) {
-        return;
-      }
-      duplicates.push(item.imageName);
+    // Don't download texture images twice
+    const imageName = processed[hash];
+    if (imageName !== undefined) {
+      item.imageName = imageName;
+      return;
+    }
+
+    processed[hash] = item.imageName;
+
+    // Check for an already exisitng file and fall back to item name
+    if (existsSync(filePath)) {
+      const [_, ext] = item.imageName.split('.');
+      /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
+      item.imageName = sanitize(`${item.name.replace(/[ /*]/g, '-')}.${ext!}`);
+      processed[hash] = item.imageName;
+      filePath = path.join(basePath, item.imageName);
     }
 
-    // Check if the previous image was for a component because they might
-    // have different naming schemes like lex-prime
-    if (!cached || cached.hash !== hash || cached.isComponent !== isComponent) {
+    if (cached?.hash !== hash) {
       try {
         const retry = (err: Error & { code?: string }): Promise<Buffer | string> => {
           if (err.code === 'ENOTFOUND') {
@@ -268,6 +287,8 @@ class Build {
       } catch (e) {
         // swallow error
         console.error(e);
+        item.imageName = 'missing.png';
+        throw e;
       }
     }
   }

build/parser.ts

@@ -1,5 +1,3 @@
-import { createHash } from 'node:crypto';
-
 import cloneDeep from 'lodash.clonedeep';
 import sanitize from 'sanitize-filename';
 
@@ -567,79 +565,27 @@ class Parser {
       return;
     }
 
-    const encode = (str: string): string =>
-      sanitize(
-        str
-          .replace('/', '')
-          .replace(/[ /*]/g, '-')
-          .replace(/[:<>[\]?!"]/g, '')
-          .toLowerCase()
-      );
-
-    const imageStub = image.textureLocation;
-    const parts = imageStub.split('.');
-    const ext = (parts[parts.length - 1] ?? 'png').replace(/\?!.*/, '').replace(/!.*$/, ''); // .png, .jpg, etc
-    const hash = (str: string): string => createHash('sha256').update(str).digest('hex');
-
     // Enforce arcane and blueprint image name
-    if (item.name === 'Arcane') {
-      item.imageName = `arcane.${ext}`;
+    if (item.name === 'Arcane' || item.name === 'Blueprint') {
+      if (item.name === 'Arcane') item.imageName = `arcane.png`;
+      if (item.name === 'Blueprint') item.imageName = `blueprint.png`;
       return;
     }
-    if (item.name === 'Blueprint') {
-      item.imageName = `blueprint.${ext}`;
-      return;
-    }
-
-    // Turn any separators into dashes and remove characters that would break
-    // the filesystem.
-    item.imageName = encode(item.name);
-
-    if (item.type === 'Nightwave Challenge') {
-      const name = item.name.replace(/\sM{0,3}(CM|CD|D?C{0,3})?(XC|XL|L?X{0,3})?(IX|IV|V?I{0,3})?$/i, '').trim();
-      const sterilized = item.uniqueName.replace(/[0-9]{1,3}$/, '');
-      item.imageName = `${encode(name)}-${hash(sterilized).slice(0, 10)}.${ext}`;
 
+    const imageStub = image.textureLocation;
+    const uniqueName = imageStub.split('!')[0];
+    if (uniqueName === undefined) {
+      warnings.missingImage.push(item.name);
       return;
     }
 
-    // Components usually have the same generic images, so we should remove the
-    // parent name here. Note that there's a difference between prime/non-prime
-    // components, so we'll keep the prime in the name.
-    if (item.parent) {
-      item.imageName = item.imageName.replace(`${encode(item.parent)}-`, '');
-      if (item.name.includes('Prime')) {
-        item.imageName = `prime-${item.imageName}`;
-        // check if the image name ends with prime as some older prime secondaries use the full parent name
-        if (item.imageName.endsWith('prime')) item.imageName = item.imageName.replace(/-prime$/, '');
-      }
-    }
-
-    // Relics should use the same image based on type, as they all use the same.
-    // The resulting format looks like `axi-intact`, `axi-radiant`
-    if (item.type === 'Relic') {
-      item.imageName = item.imageName.replace(/-(.*?)-/, '-'); // Remove second word (type)
-    }
-
-    // Remove the mark number and house name from Railjack weapons
-    if (item.productCategory === 'CrewShipWeapons') {
-      item.imageName = item.imageName.replace(/(lavan|vidar|zetki)-|(-mk-i+)/g, '');
-
-      // Add original file extension
-      item.imageName += `.${ext}`;
+    const imageName = uniqueName.split('/').reverse()[0];
+    if (imageName === undefined) {
+      warnings.missingImage.push(item.name);
       return;
     }
 
-    // Some items have the same name - so add a partial hash as an identifier
-    // but avoid making component images different
-    //
-    // Regex avoids Warframe componenets and Necramech weapons and suit
-    if (item.type !== 'Relic' && !/Recipes|(Resources\/Mechs)/.test(item.uniqueName)) {
-      item.imageName += `-${hash(item.uniqueName).slice(0, 10)}`;
-    }
-
-    // Add original file extension
-    item.imageName += `.${ext}`;
+    item.imageName = sanitize(imageName);
   }
 
   /**

test/index.spec.mjs

@@ -1,11 +1,13 @@
 import assert from 'node:assert';
-import { resolve } from 'node:path';
+import path, { resolve } from 'node:path';
 import { createRequire } from 'module';
 
 import gc from 'expose-gc/function.js';
 import { expect } from 'chai';
 
 import dedupe from '../build/dedupe';
+import { statSync, existsSync } from 'node:fs';
+import sharp from 'sharp';
 
 const require = createRequire(import.meta.url);
 const masterableCategories = require('../config/masterableCategories.json');
@@ -402,6 +404,53 @@ const test = (base) => {
         });
       });
     });
+    describe('images', async () => {
+      let imageNames;
+      beforeEach(async () => {
+        const items = await wrapConstr({ category: ['All'] });
+        imageNames = items
+          .filter((item) => item && item.category !== 'Enemy')
+          .flatMap((item) => {
+            const result = [item];
+            if (Array.isArray(item?.components)) {
+              result.push(...item.components);
+            }
+            return result;
+          })
+          .filter((item) => typeof item.imageName === 'string');
+      });
+      it('should have the image stored on disk', () => {
+        for (const { imageName, uniqueName } of imageNames) {
+          if(imageName === 'missing.png') continue;
+          const imagePath = path.join('./data/img/', imageName);
+          const nameInfo = `in { uniqueName: ${uniqueName}, imageName: ${imageName} }`;
+          let exists = existsSync(imagePath);
+          assert(exists, `${imageName} should exist ${nameInfo}`);
+          if (exists) {
+            const { size } = statSync(imagePath, () => {});
+            assert(size > 0, `size should be greater than zero ${nameInfo}`);
+          }
+        }
+      });
+      it('size should be greater than 0', () => {
+        for (const { imageName, uniqueName } of imageNames) {
+          if(imageName === 'missing.png') continue;
+          const imagePath = path.join('./data/img/', imageName);
+          const nameInfo = `in { uniqueName: ${uniqueName}, imageName: ${imageName} }`;
+          const { size } = statSync(imagePath, () => {});
+          assert(size > 0, `size should be greater than zero ${nameInfo}`);
+        }
+      });
+      it('should be a valid image', async () => {
+        for (const { imageName } of imageNames) {
+          if(imageName === 'missing.png') continue;
+          const imagePath = path.join('./data/img/', imageName);
+          await sharp(imagePath)
+            .metadata()
+            .catch((e) => assert.fail(e));
+        }
+      });
+    });
   });
 };
 

test/utilities/find.spec.mjs

@@ -58,7 +58,7 @@ describe('#loadMods', () => {
       uniqueName: '/Lotus/Upgrades/Mods/Randomized/LotusRifleRandomModRare',
       polarity: 'Vazarin',
       rarity: 'Common',
-      imageName: 'rifle-riven-mod-e05c5519f1.png',
+      imageName: 'OmegaMod.png',
       category: 'Mods',
       buffs: [
         { tag: 'WeaponCritDamageMod', val: 0.3296302411049886 },