Code Commits61
t
Download ZIP
572 lines28 KBJavaScript
1/*
2make-catalog.mjs: builds the PS3 World app's catalogue under catalog/v1/:
3
4 catalog.json the index: each section's lists, their pages (with
5 sha256s), counts and where each letter starts
6 homebrew/<category>/N.json homebrew, one list per site category, 100 items a page
7 theme/<list>/N.json themes: all of them, then one list per site category
8 avatar/<list>/N.json avatars: all of them, then one list per region, as packs (folders):
9 avatar/<list>/packs/<pack>/ each pack's avatars, a list of its own (its item's "list")
10 <list>/search.txt "title<TAB>author<TAB>group" a line, for searching a list
11 icons/<id>.jpg 128 px JPEG icons
12 previews/<id>.jpg homebrew's 480x270 previews (themes' and avatars' are the
13 site's own, already on dl.ps3.world beside each download)
14
15 node make-catalog.mjs everything
16 node make-catalog.mjs --limit 3 --out <dir> the first 3 of each type (a quick sample)
17 node make-catalog.mjs --only theme --out <dir> one type (homebrew, theme, avatar; comma separated)
18 (both need --out, so they can't replace the full catalogue)
19 node make-catalog.mjs --out <dir> write somewhere else (the default is CONFIG.outDir)
20 node make-catalog.mjs --verify re-hash every page and file the written index lists
21
22Reads the site build data (themes.json, avatars.json) and homebrew.manifest.json,
23hashes the local copy of every download (the same files that are on R2), and
24writes into OUT_DIR. It never uploads: it prints the rclone command to run.
25Hashes are cached by path, size and modified time, so a second run is quick.
26*/
27import fs from 'node:fs';
28import fsp from 'node:fs/promises';
29import path from 'node:path';
30import crypto from 'node:crypto';
31import { fileURLToPath } from 'node:url';
32import sharp from 'sharp';
33
34const HERE = path.dirname(fileURLToPath(import.meta.url));
35
36// ---- where things are on this PC ----
37const CONFIG = {
38 themesJson: 'G:/ps3-themes/_site/data/themes.json',
39 avatarsJson: 'G:/PS3 Avatars/_site_avatars/data/avatars.json',
40 manifest: path.join(HERE, 'homebrew.manifest.json'),
41 homebrewRoot: 'D:/PS3 WORLD R2', // manifest "file" paths are relative to this (and to the bucket)
42 // the app's update in the index ("app"): off while 1.2.1 is about, whose Update screen freezes the PS3
43 // (fixed in 1.2.2); it updates from its Homebrew > Tools item meanwhile. Back on when most have 1.2.2
44 offerAppUpdate: false,
45 appFolder: 'homebrewtools', // under homebrewRoot (and the bucket): PS3WorldApp-v<version>.pkg, the newest offered as the app's update
46 siteRoot: 'G:/PS3 World Self Host Ver - Copy', // manifest "icon" paths are relative to this
47 // a dl.ps3.world path's first folder -> where those files are on this PC
48 roots: {
49 bestps3themes: 'G:/ps3-themes/Best-PS3-Themes/PS3_Themes',
50 ps3themes: 'G:/ps3-themes/PS3_Themes',
51 ps3avatars: 'D:/PS3 Avatars',
52 },
53 outDir: 'G:/ps3world-catalog',
54 baseUrl: 'https://dl.ps3.world/',
55 bucket: 'R2:ps3wrld-content',
56 includeNsfw: false,
57};
58
59const SCHEMA = 3; // 2: an index (catalog.json) and lists in pages of PAGE_SIZE items
60 // 3: items can be folders (avatar packs) with a list of their own
61const CATALOG_DIR = 'catalog/v1'; // on dl.ps3.world and under outDir
62const PAGE_SIZE = 100;
63const TYPES = ['homebrew', 'theme', 'avatar'];
64// homebrew is split into the site's categories (the manifest's "group"), in this order
65const HOMEBREW_LISTS = ['Games', 'Ports', 'Edits', 'Tools', 'Media', 'Misc'];
66const ICON_SIZE = 128;
67const HOMEBREW_ICON = 256; // (homebrew's few are drawn up to 300 px at 1080p)
68const ICON_BG = { r: 0x1a, g: 0x1c, b: 0x20 };
69const HASH_JOBS = 4;
70
71// ---- arguments ----
72const args = process.argv.slice(2);
73const argVal = name => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; };
74const LIMIT = argVal('--limit') ? parseInt(argVal('--limit'), 10) : 0;
75const ONLY = (argVal('--only') || 'homebrew,theme,avatar').split(',');
76const VERIFY = args.includes('--verify');
77if (argVal('--out')) CONFIG.outDir = path.resolve(argVal('--out'));
78else if (LIMIT || argVal('--only')) {
79 // a partial catalogue in the live folder would go up with the next rclone copy
80 console.error('--limit and --only need --out <dir>: they would overwrite the full catalogue in ' + CONFIG.outDir);
81 process.exit(1);
82}
83
84const warnings = [];
85const warn = msg => { warnings.push(msg); };
86
87// ---- paths ----
88
89// "bestps3themes/a%20b/c.p3t" -> local file, via CONFIG.roots
90function localFor(relUrl) {
91 const parts = relUrl.split('/').map(s => decodeURIComponent(s));
92 const root = CONFIG.roots[parts[0]];
93 return root ? path.join(root, ...parts.slice(1)) : null;
94}
95
96function relFromUrl(url) {
97 if (!url || !url.startsWith(CONFIG.baseUrl)) return null;
98 return url.slice(CONFIG.baseUrl.length);
99}
100
101const encodePath = p => p.split('/').map(encodeURIComponent).join('/');
102
103const sha256Of = s => crypto.createHash('sha256').update(s).digest('hex');
104
105// the letter the app files a title under (src/ui.c Letter_Of): the first ASCII
106// letter, '#' for a digit or anything non-ASCII first; other punctuation is skipped
107function letterOf(title) {
108 for (const ch of title) {
109 if (/[a-z]/i.test(ch)) return ch.toUpperCase();
110 if (/[0-9]/.test(ch) || ch.charCodeAt(0) >= 0x80) return '#';
111 }
112 return '#';
113}
114const letterRank = t => { const l = letterOf(t); return l === '#' ? 0 : l.charCodeAt(0) - 64; };
115
116// one list: A-Z (or in the order given), written as pages of PAGE_SIZE and a search.txt; returns its index entry
117function writeList(name, list, relDir, order) {
118 list.sort(order || ((a, b) => letterRank(a.title) - letterRank(b.title) ||
119 a.title.localeCompare(b.title, 'en', { sensitivity: 'base', numeric: true }) || a.id.localeCompare(b.id)));
120 const dir = path.join(CONFIG.outDir, relDir);
121 fs.mkdirSync(dir, { recursive: true });
122 const entry = { name, count: list.length, pages: [], search: null, letters: {} };
123 list.forEach((s, i) => { const l = letterOf(s.title); if (!(l in entry.letters)) entry.letters[l] = i; });
124 for (let p = 0; p * PAGE_SIZE < list.length; p++) {
125 const bad = list.slice(p * PAGE_SIZE, (p + 1) * PAGE_SIZE).find(s => !ID_RE.test(s.id));
126 if (bad) throw new Error(`id "${bad.id}" (${bad.title}) isn't one the app accepts`);
127 const page = {
128 schema: SCHEMA, page: p + 1,
129 items: list.slice(p * PAGE_SIZE, (p + 1) * PAGE_SIZE).map(s => ({
130 id: s.id, type: s.type, title: s.title, author: s.author, version: s.version,
131 description: s.description, group: s.group, size: s.size, sha256: s.sha256,
132 url: s.url, icon: s.icon, preview: s.preview, requires: s.requires, list: s.list,
133 titleId: s.titleId || undefined, // (homebrew: where it installs; 1.2.1 on)
134 updated: s.updated || undefined, // (homebrew: when its PKG last changed, seconds since 1970)
135 })),
136 };
137 const json = JSON.stringify(page);
138 fs.writeFileSync(path.join(dir, `${p + 1}.json`), json);
139 entry.pages.push({ url: `${relDir}/${p + 1}.json`, sha256: sha256Of(json), size: Buffer.byteLength(json) });
140 }
141 // one line an item, in list order: what the app searches without loading every page
142 const clean = s => (s || '').replace(/[\t\r\n]+/g, ' ');
143 const search = list.map(s => `${clean(s.title)}\t${clean(s.author)}\t${clean(s.group)}\n`).join('');
144 fs.writeFileSync(path.join(dir, 'search.txt'), search);
145 entry.search = { url: `${relDir}/search.txt`, sha256: sha256Of(search), size: Buffer.byteLength(search) };
146 return entry;
147}
148
149// the site builds write JSON from PowerShell, with a byte-order mark
150const readJson = file => JSON.parse(fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, ''));
151
152// ---- hashing, cached ----
153const cachePath = path.join(CONFIG.outDir, '.cache', 'hashes.json');
154let hashCache = {};
155try { hashCache = JSON.parse(fs.readFileSync(cachePath, 'utf8')); } catch {}
156
157async function fileInfo(file, useCache = true) {
158 let st;
159 try { st = await fsp.stat(file); } catch { return null; }
160 const key = `${file}|${st.size}|${st.mtimeMs}`;
161 if (useCache && hashCache[key]) return { size: st.size, sha256: hashCache[key] };
162 const sha256 = await new Promise((resolve, reject) => {
163 const h = crypto.createHash('sha256');
164 fs.createReadStream(file).on('data', d => h.update(d)).on('end', () => resolve(h.digest('hex'))).on('error', reject);
165 });
166 hashCache[key] = sha256;
167 return { size: st.size, sha256 };
168}
169
170// runs fn over items, n at a time, with a progress line
171async function pool(items, n, label, fn) {
172 let next = 0, done = 0;
173 const tick = () => { if (process.stdout.isTTY) process.stdout.write(`\r${label}: ${done}/${items.length}`); };
174 await Promise.all(Array.from({ length: n }, async () => {
175 while (next < items.length) {
176 const i = next++;
177 await fn(items[i]);
178 done++;
179 if (done % 50 === 0) tick();
180 }
181 }));
182 tick();
183 if (process.stdout.isTTY) process.stdout.write('\n');
184}
185
186// ---- icons ----
187// (homebrew's are bigger, in icons/hd: sharp at 1080p, and a new address so apps fetch them again)
188async function makeIcon(src, id, size = ICON_SIZE, dir = 'icons') {
189 if (!src) return '';
190 try { await fsp.access(src); } catch { warn(`icon missing for ${id}: ${src}`); return ''; }
191 const rel = `${CATALOG_DIR}/${dir}/${id}.jpg`;
192 const out = path.join(CONFIG.outDir, rel);
193 try {
194 const [a, b] = await Promise.all([fsp.stat(src), fsp.stat(out).catch(() => null)]);
195 if (b && b.mtimeMs >= a.mtimeMs) return rel; // made already
196 await fsp.mkdir(path.dirname(out), { recursive: true });
197 // the PS3 shows JPEG and PNG only: baseline JPEG, letterboxed into a square
198 // (failOn none: a few avatar PNGs have damaged chunks but still decode)
199 await sharp(src, { failOn: 'none' }).resize(size, size, { fit: 'contain', background: ICON_BG })
200 .flatten({ background: ICON_BG })
201 .jpeg({ quality: 80, progressive: false, mozjpeg: false, chromaSubsampling: '4:2:0' })
202 .toFile(out);
203 return rel;
204 } catch (e) {
205 warn(`icon failed for ${id}: ${e.message}`);
206 return '';
207 }
208}
209
210// homebrew's preview: its site image, letterboxed into 960x540 (the detail screen's box at 1080p)
211async function makePreview(src, id) {
212 if (!src) return '';
213 try { await fsp.access(src); } catch { return ''; }
214 const rel = `${CATALOG_DIR}/previews/hd/${id}.jpg`;
215 const out = path.join(CONFIG.outDir, rel);
216 try {
217 const [a, b] = await Promise.all([fsp.stat(src), fsp.stat(out).catch(() => null)]);
218 if (b && b.mtimeMs >= a.mtimeMs) return rel;
219 await fsp.mkdir(path.dirname(out), { recursive: true });
220 await sharp(src, { failOn: 'none' }).resize(960, 540, { fit: 'contain', background: ICON_BG })
221 .flatten({ background: ICON_BG })
222 .jpeg({ quality: 82, progressive: false, mozjpeg: false, chromaSubsampling: '4:2:0' })
223 .toFile(out);
224 return rel;
225 } catch (e) {
226 warn(`preview failed for ${id}: ${e.message}`);
227 return '';
228 }
229}
230
231// ---- ids ----
232// the app (src/catalog.c Id_Ok) skips any other id: it names cache files
233const ID_RE = /^[a-z0-9][a-z0-9-]{0,62}$/;
234const slug = (base, max) => base.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, max).replace(/-+$/, '');
235const usedIds = new Set();
236function uniqueId(base) {
237 const clean = base.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').slice(0, 60);
238 let id = clean, n = 2;
239 while (usedIds.has(id)) id = `${clean}-${n++}`;
240 usedIds.add(id);
241 return id;
242}
243
244// a PKG's title ID ("PSWD00001"), from the content ID in its header ("UP0001-PSWD00001_00-..."): the
245// folder it installs to under /dev_hdd0/game, so the app can tell it's installed; "" if it isn't a PKG
246function pkgTitleId(file) {
247 let fd;
248 try {
249 fd = fs.openSync(file, 'r');
250 const head = Buffer.alloc(0x54);
251 if (fs.readSync(fd, head, 0, head.length, 0) < head.length || head.readUInt32BE(0) !== 0x7f504b47) return '';
252 const m = /^[A-Z0-9]{6}-([A-Z0-9]{9})_\d\d-/.exec(head.toString('latin1', 0x30, 0x54));
253 return m ? m[1] : '';
254 } catch { return ''; }
255 finally { if (fd !== undefined) fs.closeSync(fd); }
256}
257
258// ---- sources: each returns { item fields..., _file, _icon } ----
259function homebrewSources() {
260 const m = readJson(CONFIG.manifest);
261 return m.items.map(e => ({
262 id: uniqueId(e.id || 'hb-' + path.basename(e.file, '.pkg')), // (an entry's own "id" lasts past a new file name)
263 type: 'homebrew',
264 title: e.title,
265 author: e.author || '',
266 version: e.version || '',
267 description: e.description || '',
268 group: e.group || '',
269 url: encodePath(e.file),
270 requires: e.requires || 'hen',
271 titleId: pkgTitleId(path.join(CONFIG.homebrewRoot, e.file)),
272 _file: path.join(CONFIG.homebrewRoot, e.file),
273 _icon: e.icon ? path.join(CONFIG.siteRoot, e.icon) : '',
274 }));
275}
276
277function themeSources() {
278 const list = readJson(CONFIG.themesJson);
279 const out = [];
280 for (const t of list) {
281 if (t.nsfw && !CONFIG.includeNsfw) continue;
282 const rel = relFromUrl(t.p3t);
283 if (!rel) { warn(`theme ${t.slug}: download isn't on ${CONFIG.baseUrl}`); continue; }
284 const img = relFromUrl(t.image);
285 out.push({
286 id: uniqueId(`th-${t.source}-${t.slug}`),
287 type: 'theme',
288 title: t.title,
289 author: t.author || '',
290 version: '',
291 description: `${t.sourceName} · ${(t.categories || [t.category]).join(', ')}`,
292 group: t.category || '',
293 _list: t.category || 'Misc',
294 url: rel,
295 requires: 'any',
296 _file: localFor(rel),
297 _icon: img && t.hasImage ? localFor(img) : '',
298 _preview: img && t.hasImage ? img : '',
299 });
300 }
301 return out;
302}
303
304function avatarSources() {
305 const list = readJson(CONFIG.avatarsJson);
306 const perPack = {};
307 const out = [];
308 for (const a of list) {
309 const rel = relFromUrl(a.dl);
310 if (!rel) { warn(`avatar ${a.pack}/${a.title}: no download on ${CONFIG.baseUrl}`); continue; }
311 const n = perPack[a.pack] = (perPack[a.pack] || 0) + 1;
312 const img = relFromUrl(a.image);
313 out.push({
314 id: uniqueId(`av-${path.basename(decodeURIComponent(rel), '.edat').replace(/^PSNA_/, '').slice(0, 16)}`),
315 type: 'avatar',
316 title: `${a.packName} #${n}`,
317 author: '',
318 version: '',
319 description: `Avatar ${a.title} from ${a.packName} (${[a.region, a.titleId].filter(Boolean).join(', ')})`,
320 group: a.packName,
321 _list: !a.region || a.region === 'Unknown' ? 'Other' : a.region,
322 _pack: a.pack,
323 url: rel,
324 requires: 'hen',
325 _file: localFor(rel),
326 _icon: img ? localFor(img) : '',
327 _preview: img || '',
328 });
329 }
330 return out;
331}
332
333// ---- when each item first appeared: "Recently added" is the last RECENT_DAYS ----
334// (the first run marks everything there as "before": the list starts with what's added after it)
335const RECENT_DAYS = 30;
336let seen = {};
337const seenOf = i => i._seen || seen[i.id] || 'before';
338const newestFirst = (a, b) => seenOf(b).localeCompare(seenOf(a)) || a.title.localeCompare(b.title, 'en', { sensitivity: 'base', numeric: true });
339const largestFirst = (a, b) => b.size - a.size || a.title.localeCompare(b.title, 'en', { sensitivity: 'base', numeric: true });
340
341// a section's list, and the same in two more orders the app can switch to (Select): newest
342// first, largest first. Each is a list of its own (pages, search.txt) under the list's "sorted";
343// apps before 1.2.0 don't look there
344function topList(name, rows, relDir) {
345 const entry = writeList(name, rows, relDir);
346 entry.sorted = [writeList(name, [...rows], `${relDir}/newest`, newestFirst), writeList(name, [...rows], `${relDir}/largest`, largestFirst)];
347 return entry;
348}
349
350// avatars as folders: an item per pack, whose "list" (the index's list entry, as a string)
351// is the pack's avatars in pages of their own under relDir/packs/<pack>
352function packFolders(avatars, relDir) {
353 const packs = {};
354 for (const a of avatars) (packs[a._pack] ||= []).push(a);
355 return Object.values(packs).map(av => {
356 const entry = writeList(av[0].group, av, `${relDir}/packs/${av[0]._pack}`);
357 return {
358 id: slug(`pk-${av[0]._pack}`, 63), // the same in every list the pack is in
359 type: 'avatar',
360 title: av[0].group,
361 author: '',
362 version: '',
363 description: `${av.length} avatar${av.length === 1 ? '' : 's'}`,
364 group: [...new Set(av.map(a => a._list))].join(', '),
365 size: av.reduce((n, a) => n + a.size, 0),
366 sha256: entry.pages[0].sha256, // (its first page: a folder has no file of its own)
367 url: entry.pages[0].url,
368 icon: (av.find(a => a.icon) || {}).icon || '',
369 requires: 'hen',
370 list: JSON.stringify(entry),
371 _seen: av.map(seenOf).sort().pop(), // (its newest avatar's, for "Recently added")
372 };
373 });
374}
375
376// ---- the app's own update: the newest PS3WorldApp-v<version>.pkg in appFolder, or null ----
377async function appUpdate() {
378 const dir = path.join(CONFIG.homebrewRoot, CONFIG.appFolder);
379 let names;
380 try { names = await fsp.readdir(dir); } catch { warn(`no ${dir}: the catalogue offers no app update`); return null; }
381 const ver = v => v.split('.').map(Number);
382 const newer = (a, b) => { for (let i = 0; i < 3; i++) if ((a[i] || 0) !== (b[i] || 0)) return (a[i] || 0) > (b[i] || 0); return false; };
383 let best = null;
384 for (const n of names) {
385 const m = /^PS3World(?:App)?-v?(\d+\.\d+\.\d+)\.pkg$/i.exec(n); // PS3WorldApp-v1.0.0.pkg (or PS3World-1.0.0.pkg)
386 if (m && (!best || newer(ver(m[1]), ver(best.version)))) best = { version: m[1], file: n };
387 }
388 if (!best) { warn(`no PS3WorldApp-v<version>.pkg in ${dir}: the catalogue offers no app update`); return null; }
389 const info = await fileInfo(path.join(dir, best.file));
390 return { version: best.version, url: encodePath(`${CONFIG.appFolder}/${best.file}`), size: info.size, sha256: info.sha256, _file: path.join(dir, best.file) };
391}
392
393// ---- build ----
394async function build() {
395 await fsp.mkdir(path.join(CONFIG.outDir, CATALOG_DIR, 'icons'), { recursive: true });
396 await fsp.mkdir(path.dirname(cachePath), { recursive: true });
397
398 let sources = [];
399 const pick = list => (LIMIT ? list.slice(0, LIMIT) : list);
400 if (ONLY.includes('homebrew')) sources.push(...pick(homebrewSources()));
401 if (ONLY.includes('theme')) sources.push(...pick(themeSources()));
402 if (ONLY.includes('avatar')) sources.push(...pick(avatarSources()));
403
404 const items = [];
405 await pool(sources, HASH_JOBS, 'hashing', async s => {
406 const info = s._file && await fileInfo(s._file);
407 if (!info) { warn(`${s.type} ${s.id}: file not found: ${s._file}`); return; }
408 s.size = info.size;
409 s.sha256 = info.sha256;
410 s.icon = s.type === 'homebrew' ? await makeIcon(s._icon, s.id, HOMEBREW_ICON, 'icons/hd') : await makeIcon(s._icon, s.id);
411 s.preview = s.type === 'homebrew' ? await makePreview(s._icon, s.id) : s._preview || '';
412 items.push(s);
413 });
414 const app = ONLY.includes('homebrew') ? await appUpdate() : null;
415 fs.writeFileSync(cachePath, JSON.stringify(hashCache));
416
417 // when each homebrew's PKG last changed: by title ID, so a new version under a new file name counts (by id
418 // for the few that share one). The app shows Update on an install from before then (1.2.1 on)
419 const verPath = path.join(CONFIG.outDir, '.cache', 'pkg-versions.json');
420 let vers = {};
421 try { if (fs.existsSync(verPath)) vers = JSON.parse(fs.readFileSync(verPath, 'utf8')); } catch { vers = {}; }
422 const tidCount = {};
423 for (const i of items) if (i.type === 'homebrew' && i.titleId) tidCount[i.titleId] = (tidCount[i.titleId] || 0) + 1;
424 const nowSec = Math.floor(Date.now() / 1000);
425 for (const i of items.filter(i => i.type === 'homebrew')) {
426 const key = i.titleId && tidCount[i.titleId] === 1 ? i.titleId : i.id;
427 if (!vers[key]) vers[key] = { sha256: i.sha256, changed: 0 }; // (new to the catalogue: nothing to update from)
428 else if (vers[key].sha256 !== i.sha256) vers[key] = { sha256: i.sha256, changed: nowSec };
429 if (vers[key].changed) i.updated = vers[key].changed;
430 }
431 fs.writeFileSync(verPath, JSON.stringify(vers, null, 1));
432
433 const seenPath = path.join(CONFIG.outDir, '.cache', 'first-seen.json');
434 let first = !fs.existsSync(seenPath);
435 try { seen = first ? {} : JSON.parse(fs.readFileSync(seenPath, 'utf8')); } catch { seen = {}; first = true; }
436 const today = new Date().toISOString().slice(0, 10);
437 for (const i of items) if (!seen[i.id]) seen[i.id] = first ? 'before' : today;
438 fs.writeFileSync(seenPath, JSON.stringify(seen));
439 const cutoff = new Date(Date.now() - RECENT_DAYS * 864e5).toISOString().slice(0, 10);
440 const isRecent = i => seen[i.id] !== 'before' && seen[i.id] >= cutoff;
441
442 // the index: each section's lists (homebrew: one per category; themes, avatars: all, then one
443 // per category or region, the biggest first, Other last), each in pages
444 const index = {
445 schema: SCHEMA,
446 generated: new Date().toISOString().replace(/\.\d+Z$/, 'Z'),
447 base_url: CONFIG.baseUrl,
448 page_size: PAGE_SIZE,
449 sections: [],
450 };
451 // (apps before 0.6.0 ignore it)
452 if (app && CONFIG.offerAppUpdate) index.app = { version: app.version, url: app.url, size: app.size, sha256: app.sha256 };
453 else if (app) warn(`the app's update (${app.version}) isn't offered: CONFIG.offerAppUpdate is off`);
454 for (const type of TYPES) {
455 const all = items.filter(i => i.type === type);
456 const section = { type, count: all.length, lists: [] };
457 if (type === 'homebrew') {
458 const names = [...HOMEBREW_LISTS, ...new Set(all.map(i => i.group).filter(g => !HOMEBREW_LISTS.includes(g)))];
459 for (const name of names) {
460 const list = all.filter(i => i.group === name);
461 if (list.length) section.lists.push(topList(name, list, `${CATALOG_DIR}/homebrew/${name.toLowerCase().replace(/[^a-z0-9]+/g, '-')}`));
462 }
463 const recent = all.filter(isRecent);
464 if (recent.length) section.lists.push(writeList('Recently added', recent, `${CATALOG_DIR}/homebrew/recent`, newestFirst));
465 } else {
466 // (avatars: a list's rows are its packs)
467 const rows = (list, relDir) => (type === 'avatar' ? packFolders(list, relDir) : list);
468 section.lists.push(topList(`All ${type}s`, rows(all, `${CATALOG_DIR}/${type}/all`), `${CATALOG_DIR}/${type}/all`));
469 const recent = all.filter(isRecent);
470 if (recent.length)
471 section.lists.push(writeList('Recently added', rows(recent, `${CATALOG_DIR}/${type}/recent`), `${CATALOG_DIR}/${type}/recent`, newestFirst));
472 const by = {};
473 for (const i of all) (by[i._list] ||= []).push(i);
474 // the biggest first (avatars: by packs, the count the app shows); "Other" (no region) last
475 const size = list => (type === 'avatar' ? new Set(list.map(i => i._pack)).size : list.length);
476 for (const [name, list] of Object.entries(by).sort((a, b) =>
477 (a[0] === 'Other') - (b[0] === 'Other') || size(b[1]) - size(a[1]))) {
478 const relDir = `${CATALOG_DIR}/${type}/${name.toLowerCase().replace(/[^a-z0-9]+/g, '-')}`;
479 section.lists.push(topList(name, rows(list, relDir), relDir));
480 }
481 }
482 index.sections.push(section);
483 }
484 // homebrew's descriptions in other languages: generator/descriptions/<code>.json (id: text),
485 // published as lang/<code>.json with what's in the catalogue; the app loads its language's
486 const descDir = path.join(HERE, 'descriptions');
487 const hb = items.filter(i => i.type === 'homebrew' && i.description);
488 const orphans = new Set();
489 for (const f of fs.existsSync(descDir) ? fs.readdirSync(descDir).filter(f => /^[a-z]{2}\.json$/.test(f)).sort() : []) {
490 const code = f.slice(0, 2), all = readJson(path.join(descDir, f)), out = {};
491 for (const i of [...hb].sort((a, b) => a.id.localeCompare(b.id))) if (all[i.id]) out[i.id] = all[i.id];
492 const missing = hb.filter(i => !all[i.id]).map(i => i.title);
493 if (missing.length) warn(`descriptions/${f}: none yet for ${missing.join(', ')} (English shows)`);
494 for (const k of Object.keys(all)) if (!hb.some(i => i.id === k)) orphans.add(k);
495 const text = JSON.stringify(out), rel = `${CATALOG_DIR}/lang/${code}.json`;
496 fs.mkdirSync(path.join(CONFIG.outDir, CATALOG_DIR, 'lang'), { recursive: true });
497 fs.writeFileSync(path.join(CONFIG.outDir, rel), text);
498 (index.descriptions ||= {})[code] = { url: rel, sha256: sha256Of(text), size: Buffer.byteLength(text) };
499 }
500 // (a new version under a new file name makes a new id: its translations stay under the old one)
501 if (orphans.size && ONLY.includes('homebrew') && !argVal('--limit')) warn(`descriptions: translations for ${[...orphans].join(', ')}, which aren't in the catalogue any more: ` +
502 `if one's a new version under a new file name, give its manifest entry the old "id"`);
503
504 const json = JSON.stringify(index);
505 const outFile = path.join(CONFIG.outDir, CATALOG_DIR, 'catalog.json');
506 fs.writeFileSync(outFile, json);
507
508 const lists = index.sections.flatMap(s => s.lists);
509 const pages = lists.flatMap(l => l.pages);
510 const count = t => items.filter(i => i.type === t).length;
511 console.log(`\nwrote ${outFile} (the index: ${(json.length / 1024).toFixed(1)} KB)`);
512 console.log(` ${items.length} items: ${count('homebrew')} homebrew, ${count('theme')} themes, ${count('avatar')} avatars`);
513 for (const s of index.sections)
514 for (const l of s.lists)
515 console.log(` ${s.type}${l.name ? '/' + l.name : ''}: ${l.count} items, ${l.pages.length} page(s), search.txt ${(l.search.size / 1024).toFixed(0)} KB`);
516 console.log(` pages: ${pages.length}, the biggest ${(Math.max(...pages.map(p => p.size)) / 1024).toFixed(1)} KB`);
517 console.log(` ${items.filter(i => !i.icon).length} without an icon`);
518 if (app) console.log(` app update offered: PS3 World ${app.version} (${app.url}, ${(app.size / 1048576).toFixed(1)} MB)`);
519 report();
520 console.log('\nUpload (you run this; copy never deletes anything on R2):');
521 if (app) // the PKG first: the index mustn't offer it before it's there
522 console.log(` rclone copy "${path.join(CONFIG.homebrewRoot, CONFIG.appFolder).replace(/\//g, '\\')}" ${CONFIG.bucket}/${CONFIG.appFolder} --progress`);
523 console.log(` rclone copy "${path.join(CONFIG.outDir, 'catalog').replace(/\//g, '\\')}" ${CONFIG.bucket}/catalog --progress`);
524}
525
526// ---- verify: every file in catalog.json against its sha256, read fresh ----
527async function verify() {
528 const file = path.join(CONFIG.outDir, CATALOG_DIR, 'catalog.json');
529 const index = readJson(file);
530 const homebrewByUrl = Object.fromEntries(homebrewSources().map(s => [s.url, s._file]));
531 let ok = 0, bad = 0;
532 // every page against the index's sha256, then every item's file against the page's
533 const cat = { items: [] };
534 const lists = index.sections.flatMap(s => s.lists);
535 for (let k = 0; k < lists.length; k++) // (a folder's own list joins the queue)
536 for (const p of lists[k].pages) {
537 const text = fs.readFileSync(path.join(CONFIG.outDir, p.url), 'utf8');
538 if (sha256Of(text) !== p.sha256) { bad++; warn(`MISMATCH page ${p.url}`); continue; }
539 for (const it of JSON.parse(text).items)
540 if (it.list) lists.push(JSON.parse(it.list));
541 else cat.items.push(it);
542 }
543 for (const f of Object.values(index.descriptions || {})) // (the translated descriptions)
544 if (sha256Of(fs.readFileSync(path.join(CONFIG.outDir, f.url), 'utf8')) !== f.sha256) { bad++; warn(`MISMATCH ${f.url}`); }
545 // the other orders: the same items, so only their pages and search files are checked
546 for (const l of index.sections.flatMap(s => s.lists).flatMap(l => l.sorted || []))
547 for (const f of [...l.pages, l.search])
548 if (sha256Of(fs.readFileSync(path.join(CONFIG.outDir, f.url), 'utf8')) !== f.sha256) { bad++; warn(`MISMATCH ${f.url}`); }
549 if (index.app) { // the app's update, checked like an item
550 homebrewByUrl[index.app.url] = path.join(CONFIG.homebrewRoot, ...index.app.url.split('/').map(decodeURIComponent));
551 cat.items.push({ id: 'ps3world-app', type: 'homebrew', ...index.app });
552 }
553 await pool(cat.items, HASH_JOBS, 'verifying', async it => {
554 const local = it.type === 'homebrew' ? homebrewByUrl[it.url] : localFor(it.url);
555 const info = local && await fileInfo(local, false);
556 if (info && info.sha256 === it.sha256 && info.size === it.size) { ok++; return; }
557 bad++;
558 warn(`MISMATCH ${it.id}: ${local} ${info ? info.sha256 + ' ' + info.size : 'missing'}`);
559 });
560 console.log(`\nverified ${cat.items.length} items: ${ok} match, ${bad} don't`);
561 report();
562 process.exitCode = bad ? 1 : 0;
563}
564
565function report() {
566 if (!warnings.length) return;
567 console.log(`\n${warnings.length} warning(s):`);
568 for (const w of warnings.slice(0, 40)) console.log(' ' + w);
569 if (warnings.length > 40) console.log(` ... and ${warnings.length - 40} more`);
570}
571
572(VERIFY ? verify() : build()).catch(e => { console.error(e); process.exit(1); });