diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index f464e92..081bfb7 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -47,7 +47,7 @@ jobs: matrix: # Add macos-latest and/or windows-latest if relevant for this package. os: [ubuntu-latest] - sdk: [2.19.0, dev] + sdk: [3.0.0, dev] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - uses: dart-lang/setup-dart@b64355ae6ca0b5d484f0106a033dd1388965d06d diff --git a/CHANGELOG.md b/CHANGELOG.md index 415ade5..925b1d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## 3.1.2-dev -- Require Dart 2.19 +- Require Dart 3.0 +- Add chunked decoding support (`startChunkedConversion`) for `CodePage` + encodings. ## 3.1.1 diff --git a/lib/src/codepage.dart b/lib/src/codepage.dart index 50941b4..c298ff5 100644 --- a/lib/src/codepage.dart +++ b/lib/src/codepage.dart @@ -277,6 +277,25 @@ CodePageDecoder _createDecoder(String characters) { return _NonBmpCodePageDecoder._(result); } +/// An input [ByteConversionSink] for decoders where each input byte can be be +/// considered independantly. +class _CodePageDecoderSink extends ByteConversionSink { + final Sink _output; + final Converter, String> _decoder; + + _CodePageDecoderSink(this._output, this._decoder); + + @override + void add(List chunk) { + _output.add(_decoder.convert(chunk)); + } + + @override + void close() { + _output.close(); + } +} + /// Code page with non-BMP characters. class _NonBmpCodePageDecoder extends Converter, String> implements CodePageDecoder { @@ -326,6 +345,10 @@ class _NonBmpCodePageDecoder extends Converter, String> } return String.fromCharCodes(buffer); } + + @override + Sink> startChunkedConversion(Sink sink) => + _CodePageDecoderSink(sink, this); } class _BmpCodePageDecoder extends Converter, String> @@ -360,6 +383,10 @@ class _BmpCodePageDecoder extends Converter, String> return String.fromCharCodes(codeUnits); } + @override + Sink> startChunkedConversion(Sink sink) => + _CodePageDecoderSink(sink, this); + String _convertAllowInvalid(List bytes) { var count = bytes.length; var codeUnits = Uint16List(count); diff --git a/pubspec.yaml b/pubspec.yaml index 5756232..77a7edb 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,7 +6,7 @@ description: >- repository: https://github.com/dart-lang/convert environment: - sdk: '>=2.19.0 <3.0.0' + sdk: '^3.0.0' dependencies: typed_data: ^1.3.0 diff --git a/test/codepage_test.dart b/test/codepage_test.dart index c0fa45f..cca75e7 100644 --- a/test/codepage_test.dart +++ b/test/codepage_test.dart @@ -2,6 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; +import 'dart:core'; import 'dart:typed_data'; import 'package:convert/convert.dart'; @@ -25,24 +27,52 @@ void main() { latinThai, latinArabic ]) { - test('${cp.name} codepage', () { - // All ASCII compatible. - for (var byte = 0x20; byte < 0x7f; byte++) { - expect(cp[byte], byte); - } - // Maps both directions. - for (var byte = 0; byte < 256; byte++) { - var char = cp[byte]; - if (char != 0xFFFD) { - var string = String.fromCharCode(char); - expect(cp.encode(string), [byte]); - expect(cp.decode([byte]), string); + group('${cp.name} codepage', () { + test('ascii compatible', () { + for (var byte = 0x20; byte < 0x7f; byte++) { + expect(cp[byte], byte); } - } - expect(() => cp.decode([0xfffd]), throwsA(isA())); - // Decode works like operator[]. - expect(cp.decode(bytes, allowInvalid: true), - String.fromCharCodes([for (var i = 0; i < 256; i++) cp[i]])); + }); + + test('bidirectional mapping', () { + // Maps both directions. + for (var byte = 0; byte < 256; byte++) { + var char = cp[byte]; + if (char != 0xFFFD) { + var string = String.fromCharCode(char); + expect(cp.encode(string), [byte]); + expect(cp.decode([byte]), string); + } + } + }); + + test('decode invalid characters not allowed', () { + expect(() => cp.decode([0xfffd]), throwsA(isA())); + }); + + test('decode invalid characters allowed', () { + // Decode works like operator[]. + expect(cp.decode(bytes, allowInvalid: true), + String.fromCharCodes([for (var i = 0; i < 256; i++) cp[i]])); + }); + + test('chunked conversion', () { + late final String decodedString; + final outputSink = StringConversionSink.withCallback( + (accumulated) => decodedString = accumulated); + final inputSink = cp.decoder.startChunkedConversion(outputSink); + final expected = StringBuffer(); + + for (var byte = 0; byte < 256; byte++) { + var char = cp[byte]; + if (char != 0xFFFD) { + inputSink.add([byte]); + expected.writeCharCode(char); + } + } + inputSink.close(); + expect(decodedString, expected.toString()); + }); }); } test('latin-2 roundtrip', () { @@ -62,14 +92,63 @@ void main() { expect(decoded, latin2text); }); - test('Custom code page', () { - var cp = CodePage('custom', "ABCDEF${"\uFFFD" * 250}"); - var result = cp.encode('BADCAFE'); - expect(result, [1, 0, 3, 2, 0, 5, 4]); - expect(() => cp.encode('GAD'), throwsFormatException); - expect(cp.encode('GAD', invalidCharacter: 0x3F), [0x3F, 0, 3]); - expect(cp.decode([1, 0, 3, 2, 0, 5, 4]), 'BADCAFE'); - expect(() => cp.decode([6, 1, 255]), throwsFormatException); - expect(cp.decode([6, 1, 255], allowInvalid: true), '\u{FFFD}B\u{FFFD}'); + group('Custom code page', () { + late final cp = CodePage('custom', "ABCDEF${"\uFFFD" * 250}"); + + test('simple encode', () { + var result = cp.encode('BADCAFE'); + expect(result, [1, 0, 3, 2, 0, 5, 4]); + }); + + test('unencodable character', () { + expect(() => cp.encode('GAD'), throwsFormatException); + }); + + test('unencodable character with invalidCharacter', () { + expect(cp.encode('GAD', invalidCharacter: 0x3F), [0x3F, 0, 3]); + }); + + test('simple decode', () { + expect(cp.decode([1, 0, 3, 2, 0, 5, 4]), 'BADCAFE'); + }); + + test('undecodable byte', () { + expect(() => cp.decode([6, 1, 255]), throwsFormatException); + }); + + test('undecodable byte with allowInvalid', () { + expect(cp.decode([6, 1, 255], allowInvalid: true), '\u{FFFD}B\u{FFFD}'); + }); + + test('chunked conversion', () { + late final String decodedString; + final outputSink = StringConversionSink.withCallback( + (accumulated) => decodedString = accumulated); + final inputSink = cp.decoder.startChunkedConversion(outputSink); + + inputSink + ..add([1]) + ..add([0]) + ..add([3]) + ..close(); + expect(decodedString, 'BAD'); + }); + + test('chunked conversion - byte conversion sink', () { + late final String decodedString; + final outputSink = StringConversionSink.withCallback( + (accumulated) => decodedString = accumulated); + final bytes = [1, 0, 3, 2, 0, 5, 4]; + + final inputSink = cp.decoder.startChunkedConversion(outputSink); + expect(inputSink, isA()); + + (inputSink as ByteConversionSink) + ..addSlice(bytes, 1, 3, false) + ..addSlice(bytes, 4, 5, false) + ..addSlice(bytes, 6, 6, true); + + expect(decodedString, 'ADA'); + }); }); }