diff --git a/build/build.ts b/build/build.ts index 11b22591d2..056abcf608 100644 --- a/build/build.ts +++ b/build/build.ts @@ -25,6 +25,9 @@ import type { CategoryData, ApiCategory, } from './types/shared'; +import { existsSync } from 'node:fs'; +import sanitize from 'sanitize-filename'; +import { createHash } from 'node:crypto'; let imageCache: CachedItem[] = []; @@ -63,9 +66,9 @@ class Build { const parsed = parser.parse(raw); const data = this.applyCustomCategories(parsed.data); const i18n = parser.applyI18n(data, raw.i18n); - const all = await this.saveJson(data, i18n); + await this.saveImages(data, raw.manifest, parsed.warnings); + await this.saveJson(data, i18n); await this.saveWarnings(parsed.warnings); - await this.saveImages(all, raw.manifest); await this.updateReadme(raw.patchlogs); // Log number of warnings at the end of the script @@ -123,7 +126,7 @@ class Build { async saveJson( categories: Record, i18n: Record>> - ): Promise { + ): Promise { let all: Item[] = []; const sort = (a: Item, b: Item): number => { if (!a.name) console.log(a); @@ -150,8 +153,6 @@ class Build { all.sort(sort); await fs.writeFile(new URL('../data/json/All.json', import.meta.url), stringify(all)); await fs.writeFile(new URL('../data/json/i18n.json', import.meta.url), JSON.stringify(JSON.parse(stringify(i18n)))); - - return all; } /** @@ -167,30 +168,40 @@ class Build { * @param items items to append images to * @param manifest image manifest to look up items from */ - async saveImages(items: Item[], manifest: ImageManifest): Promise { + async saveImages(categories: Record, manifest: ImageManifest, warnings: Warnings): Promise { // No need to go through every item if the manifest didn't change. I'm // guessing the `fileTime` key in each element works more or less like a // hash, so any change to that changes the hash of the full thing. - if (!hashManager.hasChanged('Manifest')) return; + // if (!hashManager.hasChanged('Manifest')) return; + const items = Object.values(categories).flat(); const bar = new Progress('Fetching Images', items.length); - const duplicates: string[] = []; // Don't download component images or relics twice - - for (const item of items) { - // Save image for parent item - await this.saveImage(item, false, duplicates, manifest); - // Save images for components if necessary - if (item.components) { - for (const component of item.components) { - await this.saveImage(component, true, duplicates, manifest); - } - } - // Save images for abilities - if (item.abilities) { - for (const ability of item.abilities) { - await this.saveImage(ability as Item, false, duplicates, manifest); + const processed: Record = {}; // Don't download component images or relics twice + + for (const category of Object.keys(categories)) { + const categoryData = categories[category]; + if (!categoryData) continue; + + for (const item of categoryData) { + try { + // Save image for parent item + await this.saveImage(item, false, processed, manifest); + // Save images for components if necessary + if (item.components) { + for (const component of item.components) { + await this.saveImage(component, true, processed, manifest); + } + } + // Save images for abilities + if (item.abilities) { + for (const ability of item.abilities) { + await this.saveImage(ability as Item, false, processed, manifest); + } + } + } catch { + warnings.missingImage.push(item.name); } + bar.tick(); } - bar.tick(); } // write the manifests after images have all succeeded @@ -208,41 +219,49 @@ class Build { * Download and save images for items or components. * @param item to determine and save an image for * @param isComponent whether the item is a component or a parent - * @param duplicates list of duplicated (already existing) image names + * @param processed list of duplicated (already existing) image names * @param manifest image lookup list */ - async saveImage(item: Item, isComponent: boolean, duplicates: string[], manifest: ImageManifest): Promise { - let { uniqueName } = item; - if (item.type === 'Nightwave Act') { - uniqueName = item.uniqueName.replace(/[0-9]{1,3}$/, ''); - } - - const imageBase = manifest.find((i) => i.uniqueName === uniqueName); + async saveImage( + item: Item, + isComponent: boolean, + processed: Record, + manifest: ImageManifest + ): Promise { + const imageBase = manifest.find((i) => i.uniqueName === item.uniqueName); if (!imageBase) return; const imageStub = imageBase.textureLocation.replace(/\\/g, '/').replace('xport/', ''); const imageHash = /!00_([\S]+)/.exec(imageStub); const imageUrl = `https://content.warframe.com/PublicExport/${imageStub}`; const basePath = fileURLToPath(new URL('../data/img/', import.meta.url)); - const filePath = path.join(basePath, item.imageName); - const manifestItem = manifest.find((i) => i.uniqueName === item.uniqueName); - const hash = manifestItem?.fileTime ?? imageHash?.[1] ?? undefined; + let filePath = path.join(basePath, item.imageName); + const hash = + imageBase?.fileTime ?? imageHash?.[1] ?? createHash('md5').update(imageBase.textureLocation).digest('hex'); const cached = imageCache.find((c) => c.uniqueName === item.uniqueName); // We'll use a custom blueprint image if (item.name === 'Blueprint' || item.name === 'Arcane') return; - // Don't download component images or relic images twice - if (isComponent || item.type === 'Relic') { - if (duplicates.includes(item.imageName)) { - return; - } - duplicates.push(item.imageName); + // Don't download texture images twice + const imageName = processed[hash]; + if (imageName !== undefined) { + item.imageName = imageName; + return; + } + + processed[hash] = item.imageName; + + // Check for an already exisitng file and fall back to item name + if (existsSync(filePath)) { + const [_, ext] = item.imageName.split('.'); + /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */ + item.imageName = sanitize(`${item.name.replace(/[ /*]/g, '-')}.${ext!}`); + processed[hash] = item.imageName; + filePath = path.join(basePath, item.imageName); } - // Check if the previous image was for a component because they might - // have different naming schemes like lex-prime - if (!cached || cached.hash !== hash || cached.isComponent !== isComponent) { + if (cached?.hash !== hash) { try { const retry = (err: Error & { code?: string }): Promise => { if (err.code === 'ENOTFOUND') { @@ -268,6 +287,8 @@ class Build { } catch (e) { // swallow error console.error(e); + item.imageName = 'missing.png'; + throw e; } } } diff --git a/build/parser.ts b/build/parser.ts index b445e90498..4df9450d51 100644 --- a/build/parser.ts +++ b/build/parser.ts @@ -1,5 +1,3 @@ -import { createHash } from 'node:crypto'; - import cloneDeep from 'lodash.clonedeep'; import sanitize from 'sanitize-filename'; @@ -567,79 +565,27 @@ class Parser { return; } - const encode = (str: string): string => - sanitize( - str - .replace('/', '') - .replace(/[ /*]/g, '-') - .replace(/[:<>[\]?!"]/g, '') - .toLowerCase() - ); - - const imageStub = image.textureLocation; - const parts = imageStub.split('.'); - const ext = (parts[parts.length - 1] ?? 'png').replace(/\?!.*/, '').replace(/!.*$/, ''); // .png, .jpg, etc - const hash = (str: string): string => createHash('sha256').update(str).digest('hex'); - // Enforce arcane and blueprint image name - if (item.name === 'Arcane') { - item.imageName = `arcane.${ext}`; + if (item.name === 'Arcane' || item.name === 'Blueprint') { + if (item.name === 'Arcane') item.imageName = `arcane.png`; + if (item.name === 'Blueprint') item.imageName = `blueprint.png`; return; } - if (item.name === 'Blueprint') { - item.imageName = `blueprint.${ext}`; - return; - } - - // Turn any separators into dashes and remove characters that would break - // the filesystem. - item.imageName = encode(item.name); - - if (item.type === 'Nightwave Challenge') { - const name = item.name.replace(/\sM{0,3}(CM|CD|D?C{0,3})?(XC|XL|L?X{0,3})?(IX|IV|V?I{0,3})?$/i, '').trim(); - const sterilized = item.uniqueName.replace(/[0-9]{1,3}$/, ''); - item.imageName = `${encode(name)}-${hash(sterilized).slice(0, 10)}.${ext}`; + const imageStub = image.textureLocation; + const uniqueName = imageStub.split('!')[0]; + if (uniqueName === undefined) { + warnings.missingImage.push(item.name); return; } - // Components usually have the same generic images, so we should remove the - // parent name here. Note that there's a difference between prime/non-prime - // components, so we'll keep the prime in the name. - if (item.parent) { - item.imageName = item.imageName.replace(`${encode(item.parent)}-`, ''); - if (item.name.includes('Prime')) { - item.imageName = `prime-${item.imageName}`; - // check if the image name ends with prime as some older prime secondaries use the full parent name - if (item.imageName.endsWith('prime')) item.imageName = item.imageName.replace(/-prime$/, ''); - } - } - - // Relics should use the same image based on type, as they all use the same. - // The resulting format looks like `axi-intact`, `axi-radiant` - if (item.type === 'Relic') { - item.imageName = item.imageName.replace(/-(.*?)-/, '-'); // Remove second word (type) - } - - // Remove the mark number and house name from Railjack weapons - if (item.productCategory === 'CrewShipWeapons') { - item.imageName = item.imageName.replace(/(lavan|vidar|zetki)-|(-mk-i+)/g, ''); - - // Add original file extension - item.imageName += `.${ext}`; + const imageName = uniqueName.split('/').reverse()[0]; + if (imageName === undefined) { + warnings.missingImage.push(item.name); return; } - // Some items have the same name - so add a partial hash as an identifier - // but avoid making component images different - // - // Regex avoids Warframe componenets and Necramech weapons and suit - if (item.type !== 'Relic' && !/Recipes|(Resources\/Mechs)/.test(item.uniqueName)) { - item.imageName += `-${hash(item.uniqueName).slice(0, 10)}`; - } - - // Add original file extension - item.imageName += `.${ext}`; + item.imageName = sanitize(imageName); } /** diff --git a/data/img/arcane.png b/data/img/arcane.png new file mode 100644 index 0000000000..8eb355373e Binary files /dev/null and b/data/img/arcane.png differ diff --git a/data/img/blueprint.png b/data/img/blueprint.png new file mode 100644 index 0000000000..c8fdb64225 Binary files /dev/null and b/data/img/blueprint.png differ diff --git a/test/index.spec.mjs b/test/index.spec.mjs index 74d00507a0..f24ff32f2e 100755 --- a/test/index.spec.mjs +++ b/test/index.spec.mjs @@ -1,11 +1,13 @@ import assert from 'node:assert'; -import { resolve } from 'node:path'; +import path, { resolve } from 'node:path'; import { createRequire } from 'module'; import gc from 'expose-gc/function.js'; import { expect } from 'chai'; import dedupe from '../build/dedupe'; +import { statSync, existsSync } from 'node:fs'; +import sharp from 'sharp'; const require = createRequire(import.meta.url); const masterableCategories = require('../config/masterableCategories.json'); @@ -402,6 +404,53 @@ const test = (base) => { }); }); }); + describe('images', async () => { + let imageNames; + beforeEach(async () => { + const items = await wrapConstr({ category: ['All'] }); + imageNames = items + .filter((item) => item && item.category !== 'Enemy') + .flatMap((item) => { + const result = [item]; + if (Array.isArray(item?.components)) { + result.push(...item.components); + } + return result; + }) + .filter((item) => typeof item.imageName === 'string'); + }); + it('should have the image stored on disk', () => { + for (const { imageName, uniqueName } of imageNames) { + if(imageName === 'missing.png') continue; + const imagePath = path.join('./data/img/', imageName); + const nameInfo = `in { uniqueName: ${uniqueName}, imageName: ${imageName} }`; + let exists = existsSync(imagePath); + assert(exists, `${imageName} should exist ${nameInfo}`); + if (exists) { + const { size } = statSync(imagePath, () => {}); + assert(size > 0, `size should be greater than zero ${nameInfo}`); + } + } + }); + it('size should be greater than 0', () => { + for (const { imageName, uniqueName } of imageNames) { + if(imageName === 'missing.png') continue; + const imagePath = path.join('./data/img/', imageName); + const nameInfo = `in { uniqueName: ${uniqueName}, imageName: ${imageName} }`; + const { size } = statSync(imagePath, () => {}); + assert(size > 0, `size should be greater than zero ${nameInfo}`); + } + }); + it('should be a valid image', async () => { + for (const { imageName } of imageNames) { + if(imageName === 'missing.png') continue; + const imagePath = path.join('./data/img/', imageName); + await sharp(imagePath) + .metadata() + .catch((e) => assert.fail(e)); + } + }); + }); }); }; diff --git a/test/utilities/find.spec.mjs b/test/utilities/find.spec.mjs index 79f4f79d0a..a938d3acb0 100644 --- a/test/utilities/find.spec.mjs +++ b/test/utilities/find.spec.mjs @@ -58,7 +58,7 @@ describe('#loadMods', () => { uniqueName: '/Lotus/Upgrades/Mods/Randomized/LotusRifleRandomModRare', polarity: 'Vazarin', rarity: 'Common', - imageName: 'rifle-riven-mod-e05c5519f1.png', + imageName: 'OmegaMod.png', category: 'Mods', buffs: [ { tag: 'WeaponCritDamageMod', val: 0.3296302411049886 },