Store PC protocols as deltas from the previous version

doc/protocol.md

@@ -154,3 +154,23 @@ string: ["pstring",{"countType":"varint"}]
 
 #### Compiling to JSON
 Once the protocol YAML files have been updated, run `npm run build` inside `tools/js` to generate protocol.json for each of the protocol YAML files.
+
+#### Delta protocols
+Instead of a full copy of the protocol, a version's `proto.yml` can be a **delta** on a previous version by starting with `!base: <version>` (resolved through `dataPaths.json`):
+
+```yml
+!version: 1.21.8
+!base: 1.21.6
+
+^types:
+   vec3i:            # added
+      x: varint
+      y: varint
+      z: varint
+   packet_x: ...     # redefined wholesale (replaces the base's definition)
+   OldType: !delete  # removed
+```
+
+Semantics, per section (`^types`, `^play.toClient.types`, ...): a type present in both is **replaced wholesale** by the delta's definition — there is no deep merge, because mapper values and packet IDs are positional and partial merges would be unsafe; a type only in the base is inherited; a type only in the delta is added. `TypeName: !delete` removes an inherited type, and `^section: !delete` removes a whole section. Versions whose protocol didn't change keep pointing at the previous version's data directory via `dataPaths.json`, as before.
+
+`npm run build` resolves `!base` chains automatically, so deltas compile exactly like full protocols. When updating a protocol, copy the changed/added packet definitions from the previous version's file into the new delta rather than copying the whole file.

tools/js/backfillDeltas.js

@@ -0,0 +1,177 @@
+/**
+ * One-time migration: converts full per-version proto.yml files into !base
+ * deltas (see protoDelta.js).
+ *
+ * For each consecutive pair of protocol versions (in dataPaths.json order),
+ * diffs their proto.yml at the type level and emits a minimal delta for the
+ * newer version. Before keeping anything it compiles each delta chain and
+ * verifies the result is deep-equal to the previously committed protocol.json
+ * (the golden check: key *order* may differ since added types are appended,
+ * which is not semantically meaningful, but content must match exactly).
+ *
+ * Usage:
+ *   node backfillDeltas.js [edition]        # dry run (default: pc)
+ *   node backfillDeltas.js [edition] --write
+ * After --write, run `npm run build` to regenerate the protocol.json files in
+ * the new key order.
+ */
+const fs = require('fs')
+const { join } = require('path')
+const { compile } = require('protodef-yaml')
+const { parseProto, splitSectionEntries, resolve, entryValue } = require('./protoDelta')
+
+const DATA = join(__dirname, '../../data')
+const DELETE = '!delete'
+
+// Mirror compileProtocol.js: decimal mapper keys are rendered as hex
+function visitor (key, value) {
+  if ((key === 'packet') && (value?.[1]?.[0]?.name === 'name')) {
+    const mapper = value[1][0].type[1].mappings
+    for (const k in mapper) {
+      if (k.startsWith('0x')) continue
+      mapper['0x' + Number(k).toString(16).padStart(2, '0')] = mapper[k]
+      delete mapper[k]
+    }
+  }
+  return value
+}
+
+// Distinct proto dirs in dataPaths order, with a canonical version id each
+function protoChain (edition) {
+  const dataPaths = JSON.parse(fs.readFileSync(join(DATA, 'dataPaths.json'), 'utf8'))
+  const chain = []
+  for (const [versionId, paths] of Object.entries(dataPaths[edition])) {
+    if (!paths.proto) continue
+    if (chain.length && chain[chain.length - 1].dir === paths.proto) continue
+    chain.push({ versionId, dir: paths.proto })
+  }
+  return chain
+}
+
+// Diff two parsed protos into a delta: added/changed types verbatim, removed
+// types as `TypeName: !delete`, removed sections as `^section: !delete`
+function buildDelta (prevParsed, curParsed) {
+  const sections = new Map()
+  for (const [key, curSection] of curParsed.sections) {
+    const prevSection = prevParsed.sections.get(key)
+    const prevEntries = prevSection ? splitSectionEntries(prevSection.body, `${key} (prev)`) : null
+    const curEntries = splitSectionEntries(curSection.body, `${key} (cur)`)
+    const ops = []
+    if (prevEntries) {
+      for (const typeKey of prevEntries.keys()) {
+        if (!curEntries.has(typeKey)) ops.push({ lines: [`   ${typeKey}: ${DELETE}`] })
+      }
+    }
+    for (const [typeKey, lines] of curEntries) {
+      const prev = prevEntries && prevEntries.get(typeKey)
+      if (!prev || prev.join('\n') !== lines.join('\n')) ops.push({ lines })
+    }
+    if (ops.length) sections.set(key, { header: curSection.header, ops })
+  }
+  for (const key of prevParsed.sections.keys()) {
+    if (!curParsed.sections.has(key)) sections.set(key, { header: `${key}: ${DELETE}`, ops: [] })
+  }
+  if (sections.size === 0) return null // identical protocol: keep the full file
+  return sections
+}
+
+function renderDelta (version, baseVersion, sections) {
+  const lines = [`!version: ${version}`, `!base: ${baseVersion}`]
+  for (const section of sections.values()) {
+    lines.push('', section.header)
+    if (section.header.endsWith(DELETE)) continue
+    let first = true
+    for (const op of section.ops) {
+      if (!first) lines.push('')
+      lines.push(...op.lines)
+      first = false
+    }
+  }
+  return lines.join('\n') + '\n'
+}
+
+function sortKeys (v) {
+  if (Array.isArray(v)) return v.map(sortKeys)
+  if (v && typeof v === 'object') {
+    const o = {}
+    for (const k of Object.keys(v).sort()) o[k] = sortKeys(v[k])
+    return o
+  }
+  return v
+}
+
+function main () {
+  const edition = process.argv[2] && !process.argv[2].startsWith('-') ? process.argv[2] : 'pc'
+  const write = process.argv.includes('--write')
+  const chain = protoChain(edition)
+  console.log(`${edition}: ${chain.length} distinct protocol dirs`)
+
+  let converted = 0
+  let failed = 0
+  // Full (chain-resolved) text of the previous version's protocol: diffs are
+  // always computed against the full previous protocol, never a delta file
+  let prevFullText = null
+  for (let i = 1; i < chain.length; i++) {
+    const prev = chain[i - 1]
+    const cur = chain[i]
+    const curDir = join(DATA, cur.dir)
+    const curPath = join(curDir, 'proto.yml')
+    const prevPath = join(DATA, prev.dir, 'proto.yml')
+    if (prevFullText === null) {
+      prevFullText = parseProto(fs.readFileSync(prevPath, 'utf8'), prevPath).directives.has('!base')
+        ? resolve(edition, prevPath, DATA).main
+        : fs.readFileSync(prevPath, 'utf8')
+    }
+    const original = fs.readFileSync(curPath, 'utf8')
+    const curIsDelta = parseProto(original, curPath).directives.has('!base')
+    const curFullText = curIsDelta ? resolve(edition, curPath, DATA).main : original
+    const curParsed = parseProto(curFullText, curPath)
+
+    const sections = buildDelta(parseProto(prevFullText, prevPath), curParsed)
+    if (!sections) {
+      console.log(`= ${cur.versionId}: identical to ${prev.versionId}, keeping full file`)
+      prevFullText = curFullText
+      continue
+    }
+
+    // golden check: write the delta, resolve the chain, compile, compare with
+    // the committed protocol.json, then restore unless we are writing
+    const versionLine = (curParsed.directives.get('!version') || [])[0] || ''
+    const version = entryValue(versionLine.trim())
+    if (!version) throw Error(`No !version in ${curPath}`)
+    const deltaText = renderDelta(version, prev.versionId, sections)
+    fs.writeFileSync(curPath, deltaText)
+    let err = null
+    let mergedMain = null
+    try {
+      const files = resolve(edition, curPath, DATA)
+      const json = compile(files)
+      const goldenPath = join(curDir, 'protocol.json')
+      if (fs.existsSync(goldenPath)) {
+        const expected = JSON.parse(fs.readFileSync(goldenPath, 'utf8'))
+        const actual = JSON.parse(JSON.stringify(json, visitor, 2))
+        if (JSON.stringify(sortKeys(actual)) !== JSON.stringify(sortKeys(expected))) {
+          err = 'compiled protocol differs from committed protocol.json'
+        }
+      } // else: unreleased version (pc/latest), nothing committed to compare with
+      mergedMain = files.main
+    } catch (e) {
+      err = e.stack ? e.stack.split('\n').slice(0, 4).join(' | ') : e.message
+    }
+    if (err || !write) fs.writeFileSync(curPath, original)
+    prevFullText = err ? curFullText : mergedMain
+
+    if (err) {
+      failed++
+      console.log(`✗ ${cur.versionId}: ${err}`)
+      continue
+    }
+    converted++
+    const ops = [...sections.values()].reduce((n, s) => n + s.ops.length, 0)
+    console.log(`✓ ${cur.versionId}: delta on ${prev.versionId} (${ops} type ops, ${deltaText.split('\n').length} lines)`)
+  }
+  console.log(`\n${converted} converted, ${failed} failed${write ? '' : ' (dry run: deltas NOT kept)'}`)
+  if (failed) process.exit(1)
+}
+
+try { main() } catch (e) { console.error(e); process.exit(1) }

tools/js/compileProtocol.js

@@ -5,15 +5,21 @@
 const fs = require('fs')
 const { join } = require('path')
 const { parse, compile } = require('protodef-yaml')
+const { resolveProtoDelta } = require('./protoDelta')
 
 function getJSON (path) {
   return JSON.parse(fs.readFileSync(path, 'utf-8'))
 }
 
 // Parse the YML files and turn to JSON
-function genProtoSchema (createPacketMap) {
+function genProtoSchema (createPacketMap, edition) {
+  // If proto.yml is a !base delta, resolve the chain into a full protocol
+  // file set first (no-op for full protos, which keep using the file on disk)
+  const delta = resolveProtoDelta(edition, process.cwd(), join(__dirname, '../../data'))
+  const protoInput = delta ? delta.files : './proto.yml'
+
   // Create the packet_map.yml from proto.yml
-  const parsed = parse('./proto.yml')
+  const parsed = parse(protoInput)
   const version = parsed['!version']
 
   if (createPacketMap) {
@@ -41,7 +47,12 @@ function genProtoSchema (createPacketMap) {
     fs.writeFileSync('./packet_map.yml', t)
   }
 
-  const json = compile('./proto.yml')
+  if (delta && fs.existsSync('./packet_map.yml')) {
+    // protodef-yaml only resolves !imports from disk for file inputs; feed the
+    // generated packet map into the resolved file set
+    delta.files['packet_map.yml'] = fs.readFileSync('./packet_map.yml', 'utf8')
+  }
+  const json = compile(protoInput)
   fs.rmSync('./packet_map.yml', { force: true }) // temp file
   return [version, json]
 }
@@ -59,22 +70,40 @@ function visitor (key, value) {
   return value
 }
 
+// Render object keys in sorted order so regenerated files are byte-stable
+// regardless of the source order (JSON object key order carries no meaning
+// for ProtoDef compilation). The top-level section order is preserved:
+// protodef-validator registers the global `types` namespace before the
+// per-state namespaces, so `types` must stay ahead of them no matter how the
+// source was assembled.
+function sortKeys (value, isTopLevel = false) {
+  if (Array.isArray(value)) return value.map(v => sortKeys(v))
+  if (value && typeof value === 'object') {
+    const keys = Object.keys(value)
+    if (!isTopLevel) keys.sort()
+    const out = {}
+    for (const key of keys) out[key] = sortKeys(value[key])
+    return out
+  }
+  return value
+}
+
 function convert (edition, ver, path) {
   process.chdir(path || join(__dirname, `../../data/${edition}/${ver}`))
-  const [version, json] = genProtoSchema(edition === 'bedrock')
+  const [version, json] = genProtoSchema(edition === 'bedrock', edition)
   fs.mkdirSync(`../${version}`, { recursive: true })
   if (edition === 'bedrock') {
-    fs.writeFileSync(`../${version}/protocol.json`, JSON.stringify({ types: json }, visitor, 2))
+    fs.writeFileSync(`../${version}/protocol.json`, JSON.stringify({ types: sortKeys(json, true) }, visitor, 2))
   } else if (edition === 'pc') {
-    fs.writeFileSync(`../${version}/protocol.json`, JSON.stringify(json, visitor, 2))
+    fs.writeFileSync(`../${version}/protocol.json`, JSON.stringify(sortKeys(json, true), visitor, 2))
   }
   return version
 }
 
 function validate (edition, ver, path) {
   process.chdir(path || join(__dirname, `../../data/${edition}/${ver}`))
   // console.log(process.cwd())
-  const [version, json] = genProtoSchema(edition === 'bedrock')
+  const [version, json] = genProtoSchema(edition === 'bedrock', edition)
 
   if (path.endsWith('latest')) {
     if (fs.existsSync(join(__dirname, `../../data/${edition}/${version}/proto.yml`))) {
@@ -83,8 +112,8 @@ function validate (edition, ver, path) {
   }
 
   const expected = edition === 'bedrock'
-    ? JSON.stringify({ types: json }, visitor, 2)
-    : JSON.stringify(json, visitor, 2)
+    ? JSON.stringify({ types: sortKeys(json, true) }, visitor, 2)
+    : JSON.stringify(sortKeys(json, true), visitor, 2)
 
   // If you crash here, no protocol.json was generated - run `npm run build`
   const actual = JSON.stringify(getJSON(`../${version}/protocol.json`), null, 2)