diff --git a/README.md b/README.md index 692ec63..1efc439 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ for { } ``` -## Example CLI +## Example CLIs `cmd/marcdump` is a small example program that dumps a binary MARC21 file as MARCMaker-style text (default) or MARC-in-JSON (`-json`): @@ -152,6 +152,18 @@ go run ./cmd/marcdump testdata/marc.dat go run ./cmd/marcdump -json testdata/marc.dat ``` +`cmd/marcconv` streams records between binary MARC, MARCXML, and MARC-in-JSON +(and out to MARCMaker text), converting one record at a time rather than +loading the whole input into memory. Formats are guessed from file +extensions (`.mrc`/`.marc`/`.dat`, `.xml`, `.json`, `.txt`) when not given +explicitly; use `-from`/`-to` when reading from or writing to stdin/stdout: + +```sh +go run ./cmd/marcconv testdata/marc.dat -o out.json # marc -> json, guessed from extensions +go run ./cmd/marcconv -from xml -to json < in.xml > out.json +go run ./cmd/marcconv -from marc -to text testdata/marc.dat # MARCMaker text to stdout +``` + ## Testing ```sh diff --git a/cmd/marcconv/main.go b/cmd/marcconv/main.go new file mode 100644 index 0000000..a46cb5b --- /dev/null +++ b/cmd/marcconv/main.go @@ -0,0 +1,226 @@ +// Command marcconv streams MARC21 records between binary MARC, MARCXML, and +// MARC-in-JSON (and out to MARCMaker text), converting one record at a time +// rather than loading the whole input into memory. +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + marc "github.com/beyto1974/gomarc" +) + +func main() { + from := flag.String("from", "", "input format: marc, xml, json (default: guessed from the input file's extension)") + to := flag.String("to", "", "output format: marc, xml, json, text (default: guessed from -o's extension, else text)") + out := flag.String("o", "", "output file (default: stdout)") + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: marcconv [-from marc|xml|json] [-to marc|xml|json|text] [-o output] [input]\n\n"+ + "Streams MARC21 records from input (or stdin) to the target format, one\n"+ + "record at a time. With neither -from nor -to given, formats are guessed\n"+ + "from the .mrc/.marc/.xml/.json/.txt extensions of the input file and -o.\n"+ + "Flags may appear before or after the input file.\n\n") + flag.PrintDefaults() + } + // flag.Parse stops at the first non-flag argument, so flags placed after + // the input file (e.g. "marcconv in.dat -o out.json", as shown in the + // README) would otherwise be misread as extra positional arguments. + // Reorder so flags always reach the parser regardless of where they sit. + if err := flag.CommandLine.Parse(reorderArgs(os.Args[1:])); err != nil { + os.Exit(2) + } + + if flag.NArg() > 1 { + flag.Usage() + os.Exit(2) + } + var inPath string + if flag.NArg() == 1 { + inPath = flag.Arg(0) + } + + in := io.Reader(os.Stdin) + if inPath != "" { + f, err := os.Open(inPath) + if err != nil { + fmt.Fprintln(os.Stderr, "marcconv:", err) + os.Exit(1) + } + defer func() { _ = f.Close() }() + in = f + } + + fromFmt := *from + if fromFmt == "" { + fromFmt = guessFormat(inPath) + if fromFmt == "" { + fmt.Fprintln(os.Stderr, "marcconv: -from is required (input format could not be guessed)") + os.Exit(2) + } + } + + toFmt := *to + if toFmt == "" { + toFmt = guessFormat(*out) + if toFmt == "" { + toFmt = "text" + } + } + + w := io.Writer(os.Stdout) + if *out != "" { + f, err := os.Create(*out) + if err != nil { + fmt.Fprintln(os.Stderr, "marcconv:", err) + os.Exit(1) + } + defer func() { _ = f.Close() }() + w = f + } + bw := bufio.NewWriter(w) + + err := convert(in, bw, fromFmt, toFmt) + if flushErr := bw.Flush(); flushErr != nil && err == nil { + err = flushErr + } + if err != nil { + fmt.Fprintln(os.Stderr, "marcconv:", err) + os.Exit(1) + } +} + +// reorderArgs moves recognized flag tokens (and, for flags that take a +// value, the token following them) ahead of positional arguments, so that +// flag.Parse — which stops scanning at the first non-flag token — sees them +// regardless of where the user placed them relative to the input file. +func reorderArgs(args []string) []string { + valueFlags := map[string]bool{"from": true, "to": true, "o": true} + var flags, positional []string + for i := 0; i < len(args); i++ { + a := args[i] + if a == "--" { + positional = append(positional, args[i+1:]...) + break + } + name, isFlag := strings.CutPrefix(a, "--") + if !isFlag { + name, isFlag = strings.CutPrefix(a, "-") + } + if !isFlag || name == "" { + positional = append(positional, a) + continue + } + flags = append(flags, a) + if !strings.Contains(name, "=") && valueFlags[name] && i+1 < len(args) { + i++ + flags = append(flags, args[i]) + } + } + return append(flags, positional...) +} + +// guessFormat maps a file extension to a format name, or "" if unrecognized +// (including when path is empty, e.g. stdin/stdout). +func guessFormat(path string) string { + switch strings.ToLower(filepath.Ext(path)) { + case ".mrc", ".marc", ".dat": + return "marc" + case ".xml": + return "xml" + case ".json": + return "json" + case ".txt": + return "text" + default: + return "" + } +} + +// recordSource is the shape shared by marc.Reader, marc.XMLReader, and +// marc.JSONReader. +type recordSource interface { + Next() (*marc.Record, error) +} + +// recordSink is the shape shared by marc.Writer, marc.TextWriter, +// marc.XMLWriter, and marc.JSONWriter. +type recordSink interface { + Write(*marc.Record) error +} + +func newSource(r io.Reader, format string) (recordSource, error) { + switch format { + case "marc": + return marc.NewReader(r), nil + case "xml": + return marc.NewXMLReader(r), nil + case "json": + return marc.NewJSONReader(r), nil + default: + return nil, fmt.Errorf("unsupported input format %q (want marc, xml, or json)", format) + } +} + +// newSink builds a recordSink for format, plus a close func that flushes any +// closing syntax (e.g. "", "]"). close is a no-op for formats +// that don't need one. +func newSink(w io.Writer, format string) (sink recordSink, closeFn func() error, err error) { + noop := func() error { return nil } + switch format { + case "marc": + return marc.NewWriter(w), noop, nil + case "text": + return marc.NewTextWriter(w), noop, nil + case "xml": + xw, err := marc.NewXMLWriter(w) + if err != nil { + return nil, nil, err + } + return xw, xw.Close, nil + case "json": + jw, err := marc.NewJSONWriter(w) + if err != nil { + return nil, nil, err + } + return jw, jw.Close, nil + default: + return nil, nil, fmt.Errorf("unsupported output format %q (want marc, xml, json, or text)", format) + } +} + +// convert streams every record from r (in fromFmt) to w (in toFmt). It stops +// at the first decode or write error rather than skipping bad records, since +// only the binary marc.Reader documents that it's safe to keep calling Next +// after a non-fatal error. +func convert(r io.Reader, w io.Writer, fromFmt, toFmt string) error { + src, err := newSource(r, fromFmt) + if err != nil { + return err + } + sink, closeSink, err := newSink(w, toFmt) + if err != nil { + return err + } + + count := 0 + for { + rec, err := src.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("reading record %d: %w", count+1, err) + } + count++ + if err := sink.Write(rec); err != nil { + return fmt.Errorf("writing record %d: %w", count, err) + } + } + return closeSink() +} diff --git a/marcjson.go b/marcjson.go index eaaf8ef..41fd9a7 100644 --- a/marcjson.go +++ b/marcjson.go @@ -1,6 +1,7 @@ package marc import ( + "bufio" "bytes" "encoding/json" "fmt" @@ -25,50 +26,91 @@ type jsonDataField struct { Subfields []jsonSubfield `json:"subfields"` } -// sanitizeLenientJSON escapes raw control bytes (e.g. literal newlines) found -// inside JSON string literals. Real-world MARC-in-JSON is sometimes produced -// with unescaped control characters in fixed-field data (pymarc's JSONReader -// reads it via json.load(..., strict=False), which permits this); Go's -// encoding/json has no equivalent lenient mode, so we pre-escape instead. -func sanitizeLenientJSON(data []byte) []byte { - var out bytes.Buffer - inString := false - escaped := false +// lenientJSONFilter streams data from src, escaping raw control bytes (e.g. +// literal newlines) found inside JSON string literals. Real-world +// MARC-in-JSON is sometimes produced with unescaped control characters in +// fixed-field data (pymarc's JSONReader reads it via json.load(..., +// strict=False), which permits this); Go's encoding/json has no equivalent +// lenient mode, so this pre-escapes on the fly rather than requiring the +// whole document to be buffered first. +type lenientJSONFilter struct { + src io.Reader + inString, escaped bool + pending []byte + rbuf []byte + err error +} + +func (l *lenientJSONFilter) Read(p []byte) (int, error) { + for len(l.pending) == 0 { + if l.err != nil { + return 0, l.err + } + if l.rbuf == nil { + l.rbuf = make([]byte, 4096) + } + n, err := l.src.Read(l.rbuf) + if n > 0 { + l.pending = append(l.pending, l.transform(l.rbuf[:n])...) + } + if err != nil { + l.err = err + } + } + n := copy(p, l.pending) + l.pending = l.pending[n:] + return n, nil +} + +func (l *lenientJSONFilter) transform(data []byte) []byte { + out := make([]byte, 0, len(data)) for _, b := range data { - if !inString { - out.WriteByte(b) + if !l.inString { + out = append(out, b) if b == '"' { - inString = true + l.inString = true } continue } - if escaped { - out.WriteByte(b) - escaped = false + if l.escaped { + out = append(out, b) + l.escaped = false continue } switch b { case '\\': - out.WriteByte(b) - escaped = true + out = append(out, b) + l.escaped = true case '"': - out.WriteByte(b) - inString = false + out = append(out, b) + l.inString = false case '\n': - out.WriteString(`\n`) + out = append(out, '\\', 'n') case '\r': - out.WriteString(`\r`) + out = append(out, '\\', 'r') case '\t': - out.WriteString(`\t`) + out = append(out, '\\', 't') default: if b < 0x20 { - fmt.Fprintf(&out, `\u%04x`, b) + out = append(out, []byte(fmt.Sprintf(`\u%04x`, b))...) } else { - out.WriteByte(b) + out = append(out, b) } } } - return out.Bytes() + return out +} + +// sanitizeLenientJSON is the non-streaming form of lenientJSONFilter, used by +// ParseJSON which already requires the whole document in memory. +func sanitizeLenientJSON(data []byte) []byte { + out, err := io.ReadAll(&lenientJSONFilter{src: bytes.NewReader(data)}) + if err != nil { + // bytes.Reader never returns an error other than io.EOF, which + // io.ReadAll does not surface as an error. + panic(err) + } + return out } // ParseJSON parses MARC-in-JSON data, which may be a single record object or @@ -129,6 +171,81 @@ func jsonRecordToRecord(jr jsonRecord) (*Record, error) { return r, nil } +// JSONReader iterates over records in a MARC-in-JSON document, decoding one +// record at a time rather than loading the whole document, whether it is a +// top-level array or a single bare record object. Unlike ParseJSON, it does +// not require the input to fit in memory at once. +type JSONReader struct { + br *bufio.Reader + dec *json.Decoder + isArray bool + done bool +} + +// NewJSONReader builds a JSONReader over r. +func NewJSONReader(r io.Reader) *JSONReader { + return &JSONReader{br: bufio.NewReader(&lenientJSONFilter{src: r})} +} + +// init peeks the first non-whitespace byte to determine whether the document +// is a top-level array (in which case it consumes the opening '[' so the +// decoder tracks array context for Decoder.More) or a single bare record. +func (jr *JSONReader) init() error { + for { + b, err := jr.br.ReadByte() + if err != nil { + return err + } + switch b { + case ' ', '\t', '\n', '\r': + continue + } + if err := jr.br.UnreadByte(); err != nil { + return err + } + jr.dec = json.NewDecoder(jr.br) + if b == '[' { + jr.isArray = true + if _, err := jr.dec.Token(); err != nil { + return err + } + } + return nil + } +} + +// Next decodes and returns the next record, or (nil, io.EOF) at end of document. +func (jr *JSONReader) Next() (*Record, error) { + if jr.done { + return nil, io.EOF + } + if jr.dec == nil { + if err := jr.init(); err != nil { + jr.done = true + return nil, err + } + } + + if jr.isArray { + if !jr.dec.More() { + jr.done = true + if _, err := jr.dec.Token(); err != nil { // consume closing ']' + return nil, err + } + return nil, io.EOF + } + } else { + jr.done = true + } + + var jr2 jsonRecord + if err := jr.dec.Decode(&jr2); err != nil { + jr.done = true + return nil, fmt.Errorf("marc: parsing MARC-in-JSON record: %w", err) + } + return jsonRecordToRecord(jr2) +} + // JSONWriter writes records as a MARC-in-JSON array. Close must be called to // emit the closing bracket. Ported from pymarc.writer.JSONWriter. type JSONWriter struct { diff --git a/marcjson_test.go b/marcjson_test.go index 5f5622f..7edaa22 100644 --- a/marcjson_test.go +++ b/marcjson_test.go @@ -3,6 +3,7 @@ package marc import ( "bytes" "encoding/json" + "io" "reflect" "testing" ) @@ -154,6 +155,100 @@ func TestParseJSONXMLEquivalence(t *testing.T) { } } +func TestJSONReaderArray(t *testing.T) { + data := mustReadFile(t, "test.json") + want, err := ParseJSON(data) + if err != nil { + t.Fatal(err) + } + + jr := NewJSONReader(bytes.NewReader(data)) + var got []*Record + for { + rec, err := jr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + got = append(got, rec) + } + if len(got) != len(want) { + t.Fatalf("want %d records, got %d", len(want), len(got)) + } + for i := range got { + gb, err := got[i].AsMARC() + if err != nil { + t.Fatal(err) + } + wb, err := want[i].AsMARC() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(gb, wb) { + t.Errorf("record %d: as_marc mismatch between JSONReader and ParseJSON", i) + } + } + + if _, err := jr.Next(); err != io.EOF { + t.Errorf("want io.EOF after exhausting reader, got %v", err) + } +} + +func TestJSONReaderSingleRecordNotArray(t *testing.T) { + all := mustReadFile(t, "test.json") + var original []any + if err := json.Unmarshal(sanitizeLenientJSON(all), &original); err != nil { + t.Fatal(err) + } + single, err := json.Marshal(original[0]) + if err != nil { + t.Fatal(err) + } + + jr := NewJSONReader(bytes.NewReader(single)) + rec, err := jr.Next() + if err != nil { + t.Fatal(err) + } + if _, err := jr.Next(); err != io.EOF { + t.Errorf("want io.EOF after single bare record, got %v", err) + } + + b, err := json.Marshal(rec.AsDict()) + if err != nil { + t.Fatal(err) + } + var got any + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, original[0]) { + t.Errorf("got %#v want %#v", got, original[0]) + } +} + +func TestJSONReaderLenientControlChars(t *testing.T) { + // A literal (unescaped) newline embedded in a JSON string value, as + // produced by some real-world MARC-in-JSON writers (see + // sanitizeLenientJSON/lenientJSONFilter). + raw := []byte("{\"leader\":\" 22 4500\",\"fields\":[" + + "{\"245\":{\"ind1\":\"0\",\"ind2\":\"1\",\"subfields\":[{\"a\":\"line one\nline two\"}]}}]}") + + jr := NewJSONReader(bytes.NewReader(raw)) + rec, err := jr.Next() + if err != nil { + t.Fatal(err) + } + if want, got := "line one\nline two", rec.Fields[0].Subfields[0].Value; got != want { + t.Errorf("got %q want %q", got, want) + } + if _, err := jr.Next(); err != io.EOF { + t.Errorf("want io.EOF after single bare record, got %v", err) + } +} + func TestJSONWriter(t *testing.T) { r1 := mustRecord(t) r1.AddField(NewDataField("245", "0", "1", Subfield{Code: "a", Value: "First"}))