Skip to content

Commit f8c489b

Browse files
dcodeIOdcodeIO
authored
feat: Add option to discard unknown fields (#2289)
1 parent 4b4c703 commit f8c489b

16 files changed

Lines changed: 486 additions & 177 deletions

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,9 @@ const decoded = AwesomeMessage.decode(encoded);
105105

106106
`encode` expects a message instance or equivalent plain object and does not verify input implicitly. Use `verify` for plain objects whose shape is not guaranteed, `create` to create a message instance from already valid data when useful, and `fromObject` when conversion from broader JavaScript objects is needed.
107107

108-
Plain objects can be encoded directly when they already use protobuf.js runtime types: numbers for 32-bit numeric fields, booleans for `bool`, strings for `string`, `Uint8Array` or `Buffer` for `bytes`, arrays for repeated fields, and plain objects for maps. Map keys are the string representation of the respective value or an 8-character hash string for 64-bit/`Long` keys.
108+
Plain objects can be encoded directly when they already use protobuf.js runtime types: numbers for 32-bit numeric fields, booleans for `bool`, strings for `string`, `Uint8Array` or `Buffer` for `bytes`, arrays for repeated fields, and plain objects for maps. Map keys are the string representation of the respective value or an 8-character hash string for 64-bit/`Long` keys. When exact 64-bit integer support is required, install [`long`](https://github.com/dcodeIO/long.js) with protobuf.js.
109109

110-
Install [`long`](https://github.com/dcodeIO/long.js) with protobuf.js when exact 64-bit integer support is required.
110+
Unknown fields present on the wire are preserved by default in `message.$unknowns` and forwarded when the message is re-encoded. Unknown field data can be dropped from a decoded message with `delete message.$unknowns`, discarded during decode per reader with `reader.discardUnknown = true`, or disabled by default for subsequently created readers with `Reader.discardUnknown = true`.
111111

112112
### Convert plain objects
113113

‎index.d.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1207,6 +1207,9 @@ export class Reader {
12071207
/** Read buffer length. */
12081208
len: number;
12091209

1210+
/** Whether to discard unknown fields while decoding. */
1211+
discardUnknown: boolean;
1212+
12101213
/**
12111214
* Creates a new reader using the specified buffer.
12121215
* @param buffer Buffer to read from
@@ -1329,6 +1332,9 @@ export class Reader {
13291332
/** Recursion limit. */
13301333
static recursionLimit: number;
13311334

1335+
/** Whether readers discard unknown fields while decoding. */
1336+
static discardUnknown: boolean;
1337+
13321338
/**
13331339
* Skips the next element of the specified wire type.
13341340
* @param wireType Wire type received

‎src/decoder.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,8 +186,10 @@ function decoder(mtype) {
186186
// Unknown fields
187187
gen
188188
("r.skipType(%s,q,t)", i ? "u" : "t&7")
189-
("util.makeProp(m,\"$unknowns\",false);")
190-
("(m.$unknowns||(m.$unknowns=[])).push(r.raw(s,r.pos))")
189+
("if(!r.discardUnknown){")
190+
("util.makeProp(m,\"$unknowns\",false);")
191+
("(m.$unknowns||(m.$unknowns=[])).push(r.raw(s,r.pos))")
192+
("}")
191193
("}")
192194
("if(z!==undefined)")
193195
("throw Error(\"missing end group\")");

‎src/reader.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ function Reader(buffer) {
3838
* @type {number}
3939
*/
4040
this.len = buffer.length;
41+
42+
/**
43+
* Whether to discard unknown fields while decoding.
44+
* @type {boolean}
45+
*/
46+
this.discardUnknown = Reader.discardUnknown;
4147
}
4248

4349
var create_array = typeof Uint8Array !== "undefined"
@@ -450,6 +456,12 @@ Reader.prototype.skip = function skip(length) {
450456
*/
451457
Reader.recursionLimit = util.recursionLimit;
452458

459+
/**
460+
* Whether readers discard unknown fields while decoding.
461+
* @type {boolean}
462+
*/
463+
Reader.discardUnknown = false;
464+
453465
/**
454466
* Skips the next element of the specified wire type.
455467
* @param {number} wireType Wire type received

‎tests/comp_typescript.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,11 @@ const HelloReflected = root.lookupType("Hello");
3131
const reflectedCreated = HelloReflected.create({ value: "hi" });
3232
const reflectedCreatedValue = reflectedCreated.value;
3333
const reflectedDecodedValue = HelloReflected.decode(HelloReflected.encode({ value: "hi" }).finish()).value;
34+
const reflectedReader = __1.Reader.create(HelloReflected.encode({ value: "hi" }).finish());
35+
reflectedReader.discardUnknown = true;
36+
const reflectedDecodedWithReader = HelloReflected.decode(reflectedReader);
37+
const readerDiscardUnknownDefault = __1.Reader.discardUnknown;
38+
__1.Reader.discardUnknown = readerDiscardUnknownDefault;
3439
const reflectedConvertedValue = HelloReflected.fromObject({ value: "hi" }).value;
3540
const parsedOptionValue = (_a = HelloReflected.parsedOptions) === null || _a === void 0 ? void 0 : _a[0]["(custom_option)"];
3641
const reflectedMethod = new __1.Method("Call", undefined, "Hello", "Hello", false, false, undefined, undefined, [{ option: 1 }]);
@@ -91,6 +96,9 @@ helloCreated.foo();
9196
let helloMessage = new Hello({ value: "hi" });
9297
let helloBuffer = Hello.encode(helloMessage.foo()).finish();
9398
let helloDecoded = Hello.decode(helloBuffer);
99+
let helloReader = __1.Reader.create(helloBuffer);
100+
helloReader.discardUnknown = true;
101+
let helloDecodedWithReader = Hello.decode(helloReader);
94102
// Decorators
95103
require("reflect-metadata");
96104
var AwesomeEnum;

‎tests/comp_typescript.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// test currently consists only of not throwing
22

3-
import { Root, Message, Method, Type, Field, MapField, OneOf, IEnum, IField, IMethod, IOneOf, IService, IType, ReflectedMessage, RPCImpl } from "..";
3+
import { Root, Message, Method, Type, Field, MapField, OneOf, IEnum, IField, IMethod, IOneOf, IService, IType, ReflectedMessage, Reader, RPCImpl } from "..";
44
import type { rpc as RpcNamespace } from "..";
55
import type { MyService as StaticRpcService, MyRequest as StaticRpcRequest, MyResponse as StaticRpcResponse } from "./data/rpc.d";
66

@@ -25,6 +25,11 @@ type Assert<T extends true> = T;
2525
type ReflectedTypeIsType = Assert<typeof reflectedCreated.$type extends Type ? true : false>;
2626
const reflectedCreatedValue: string = reflectedCreated.value;
2727
const reflectedDecodedValue: string = HelloReflected.decode(HelloReflected.encode({ value: "hi" }).finish()).value;
28+
const reflectedReader = Reader.create(HelloReflected.encode({ value: "hi" }).finish());
29+
reflectedReader.discardUnknown = true;
30+
const reflectedDecodedWithReader: ReflectedMessage = HelloReflected.decode(reflectedReader);
31+
const readerDiscardUnknownDefault: boolean = Reader.discardUnknown;
32+
Reader.discardUnknown = readerDiscardUnknownDefault;
2833
const reflectedConvertedValue: string = HelloReflected.fromObject({ value: "hi" }).value;
2934

3035
const parsedOptionValue: number | undefined = HelloReflected.parsedOptions?.[0]["(custom_option)"];
@@ -93,6 +98,9 @@ helloCreated.foo();
9398
let helloMessage = new Hello({ value: "hi" });
9499
let helloBuffer = Hello.encode(helloMessage.foo()).finish();
95100
let helloDecoded = Hello.decode(helloBuffer);
101+
let helloReader = Reader.create(helloBuffer);
102+
helloReader.discardUnknown = true;
103+
let helloDecodedWithReader = Hello.decode(helloReader);
96104

97105
// Decorators
98106

‎tests/comp_unknown-fields.js‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,107 @@ tape.test("unknown fields - can be discarded", function(test) {
7272
test.end();
7373
});
7474

75+
tape.test("unknown fields - reader discardUnknown option", function(test) {
76+
var root = protobuf.parse(proto).root,
77+
SimpleV1 = root.lookupType("SimpleV1"),
78+
SimpleV2 = root.lookupType("SimpleV2"),
79+
encoded = SimpleV2.encode({
80+
known: 1,
81+
futureBool: true,
82+
futureString: "hello"
83+
}).finish(),
84+
reader = protobuf.Reader.create(encoded);
85+
86+
reader.discardUnknown = true;
87+
var decoded = SimpleV1.decode(reader);
88+
89+
test.equal(decoded.known, 1, "should decode known fields");
90+
test.equal(Object.hasOwnProperty.call(decoded, "$unknowns"), false, "should not retain unknown fields");
91+
92+
var restored = SimpleV2.decode(SimpleV1.encode(decoded).finish());
93+
test.equal(restored.known, 1, "should preserve known fields");
94+
test.equal(Object.hasOwnProperty.call(restored, "futureBool"), false, "should discard unknown bool field");
95+
test.equal(Object.hasOwnProperty.call(restored, "futureString"), false, "should discard unknown string field");
96+
test.end();
97+
});
98+
99+
tape.test("unknown fields - reader discardUnknown option propagates to nested messages", function(test) {
100+
var nestedProto = "syntax = \"proto3\";\n"
101+
+ "message InnerV1 {\n"
102+
+ " int32 known = 1;\n"
103+
+ "}\n"
104+
+ "message InnerV2 {\n"
105+
+ " int32 known = 1;\n"
106+
+ " string future = 2;\n"
107+
+ "}\n"
108+
+ "message OuterV1 {\n"
109+
+ " InnerV1 inner = 1;\n"
110+
+ "}\n"
111+
+ "message OuterV2 {\n"
112+
+ " InnerV2 inner = 1;\n"
113+
+ " string future = 2;\n"
114+
+ "}\n",
115+
root = protobuf.parse(nestedProto).root,
116+
OuterV1 = root.lookupType("OuterV1"),
117+
OuterV2 = root.lookupType("OuterV2"),
118+
encoded = OuterV2.encode({
119+
inner: {
120+
known: 1,
121+
future: "nested"
122+
},
123+
future: "outer"
124+
}).finish(),
125+
reader = protobuf.Reader.create(encoded);
126+
127+
reader.discardUnknown = true;
128+
var decoded = OuterV1.decode(reader);
129+
130+
test.equal(decoded.inner.known, 1, "should decode known nested fields");
131+
test.equal(Object.hasOwnProperty.call(decoded, "$unknowns"), false, "should not retain outer unknown fields");
132+
test.equal(Object.hasOwnProperty.call(decoded.inner, "$unknowns"), false, "should not retain nested unknown fields");
133+
test.end();
134+
});
135+
136+
tape.test("unknown fields - reader discardUnknown option applies to decodeDelimited", function(test) {
137+
var root = protobuf.parse(proto).root,
138+
SimpleV1 = root.lookupType("SimpleV1"),
139+
SimpleV2 = root.lookupType("SimpleV2"),
140+
encoded = SimpleV2.encodeDelimited({
141+
known: 1,
142+
futureBool: true
143+
}).finish(),
144+
reader = protobuf.Reader.create(encoded);
145+
146+
reader.discardUnknown = true;
147+
var decoded = SimpleV1.decodeDelimited(reader);
148+
149+
test.equal(decoded.known, 1, "should decode known fields");
150+
test.equal(Object.hasOwnProperty.call(decoded, "$unknowns"), false, "should not retain unknown fields");
151+
test.end();
152+
});
153+
154+
tape.test("unknown fields - Reader.discardUnknown default", function(test) {
155+
var root = protobuf.parse(proto).root,
156+
SimpleV1 = root.lookupType("SimpleV1"),
157+
SimpleV2 = root.lookupType("SimpleV2"),
158+
discardUnknown = protobuf.Reader.discardUnknown,
159+
encoded = SimpleV2.encode({
160+
known: 1,
161+
futureBool: true
162+
}).finish();
163+
164+
try {
165+
protobuf.Reader.discardUnknown = true;
166+
167+
var decoded = SimpleV1.decode(encoded);
168+
test.equal(decoded.known, 1, "should decode known fields");
169+
test.equal(Object.hasOwnProperty.call(decoded, "$unknowns"), false, "should use the reader default");
170+
} finally {
171+
protobuf.Reader.discardUnknown = discardUnknown;
172+
}
173+
test.end();
174+
});
175+
75176
tape.test("unknown fields - only encode own unknown fields", function(test) {
76177
var root = protobuf.parse(proto).root,
77178
SimpleV1 = root.lookupType("SimpleV1"),

‎tests/data/comments.js‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,10 @@ $root.Test1 = (function() {
189189
}
190190
}
191191
reader.skipType(wireType, _depth, tag);
192-
$util.makeProp(message, "$unknowns", false);
193-
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
192+
if (!reader.discardUnknown) {
193+
$util.makeProp(message, "$unknowns", false);
194+
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
195+
}
194196
}
195197
if (_end !== undefined)
196198
throw Error("missing end group");
@@ -441,8 +443,10 @@ $root.Test2 = (function() {
441443
break;
442444
}
443445
reader.skipType(tag & 7, _depth, tag);
444-
$util.makeProp(message, "$unknowns", false);
445-
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
446+
if (!reader.discardUnknown) {
447+
$util.makeProp(message, "$unknowns", false);
448+
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
449+
}
446450
}
447451
if (_end !== undefined)
448452
throw Error("missing end group");

‎tests/data/convert.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -355,8 +355,10 @@ $root.Message = (function() {
355355
}
356356
}
357357
reader.skipType(wireType, _depth, tag);
358-
$util.makeProp(message, "$unknowns", false);
359-
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
358+
if (!reader.discardUnknown) {
359+
$util.makeProp(message, "$unknowns", false);
360+
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
361+
}
360362
}
361363
if (_end !== undefined)
362364
throw Error("missing end group");

‎tests/data/mapbox/vector_tile.js‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,10 @@ $root.vector_tile = (function() {
156156
}
157157
}
158158
reader.skipType(wireType, _depth, tag);
159-
$util.makeProp(message, "$unknowns", false);
160-
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
159+
if (!reader.discardUnknown) {
160+
$util.makeProp(message, "$unknowns", false);
161+
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
162+
}
161163
}
162164
if (_end !== undefined)
163165
throw Error("missing end group");
@@ -542,8 +544,10 @@ $root.vector_tile = (function() {
542544
}
543545
}
544546
reader.skipType(wireType, _depth, tag);
545-
$util.makeProp(message, "$unknowns", false);
546-
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
547+
if (!reader.discardUnknown) {
548+
$util.makeProp(message, "$unknowns", false);
549+
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
550+
}
547551
}
548552
if (_end !== undefined)
549553
throw Error("missing end group");
@@ -972,8 +976,10 @@ $root.vector_tile = (function() {
972976
}
973977
}
974978
reader.skipType(wireType, _depth, tag);
975-
$util.makeProp(message, "$unknowns", false);
976-
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
979+
if (!reader.discardUnknown) {
980+
$util.makeProp(message, "$unknowns", false);
981+
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
982+
}
977983
}
978984
if (_end !== undefined)
979985
throw Error("missing end group");
@@ -1415,8 +1421,10 @@ $root.vector_tile = (function() {
14151421
}
14161422
}
14171423
reader.skipType(wireType, _depth, tag);
1418-
$util.makeProp(message, "$unknowns", false);
1419-
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
1424+
if (!reader.discardUnknown) {
1425+
$util.makeProp(message, "$unknowns", false);
1426+
(message.$unknowns || (message.$unknowns = [])).push(reader.raw(start, reader.pos));
1427+
}
14201428
}
14211429
if (_end !== undefined)
14221430
throw Error("missing end group");

0 commit comments

Comments
 (0)