Avro C++

NodeImpl.hh
1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * https://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 #ifndef avro_NodeImpl_hh__
20 #define avro_NodeImpl_hh__
21
22 #include "Config.hh"
23 #include "GenericDatum.hh"
24
25 #include <iomanip>
26 #include <iostream>
27 #include <limits>
28 #include <memory>
29 #include <set>
30 #include <sstream>
31 #include <utility>
32
33 #include "Node.hh"
34 #include "NodeConcepts.hh"
35 #include "CustomAttributes.hh"
36
37 namespace avro {
38
41
42 template<
43 class NameConcept,
44 class LeavesConcept,
45 class LeafNamesConcept,
46 class MultiAttributesConcept,
47 class SizeConcept>
48 class NodeImpl : public Node {
49
50 protected:
51 explicit NodeImpl(Type type) : Node(type),
52 nameAttribute_(),
53 docAttribute_(),
54 leafAttributes_(),
55 leafNameAttributes_(),
56 customAttributes_(),
57 sizeAttribute_() {}
58
59 NodeImpl(Type type,
60 const NameConcept &name,
61 const LeavesConcept &leaves,
62 const LeafNamesConcept &leafNames,
63 const MultiAttributesConcept &customAttributes,
64 const SizeConcept &size) : Node(type),
65 nameAttribute_(name),
66 docAttribute_(),
67 leafAttributes_(leaves),
68 leafNameAttributes_(leafNames),
69 customAttributes_(customAttributes),
70 sizeAttribute_(size) {}
71
72 // Ctor with "doc"
73 NodeImpl(Type type,
74 const NameConcept &name,
75 const concepts::SingleAttribute<std::string> &doc,
76 const LeavesConcept &leaves,
77 const LeafNamesConcept &leafNames,
78 const MultiAttributesConcept &customAttributes,
79 const SizeConcept &size) : Node(type),
80 nameAttribute_(name),
81 docAttribute_(doc),
82 leafAttributes_(leaves),
83 leafNameAttributes_(leafNames),
84 customAttributes_(customAttributes),
85 sizeAttribute_(size) {}
86
87 void swap(NodeImpl &impl) {
88 std::swap(nameAttribute_, impl.nameAttribute_);
89 std::swap(docAttribute_, impl.docAttribute_);
90 std::swap(leafAttributes_, impl.leafAttributes_);
91 std::swap(leafNameAttributes_, impl.leafNameAttributes_);
92 std::swap(sizeAttribute_, impl.sizeAttribute_);
93 std::swap(customAttributes_, impl.customAttributes_);
94 std::swap(nameIndex_, impl.nameIndex_);
95 }
96
97 bool hasName() const override {
98 // e.g.: true for single and multi-attributes, false for no-attributes.
99 return NameConcept::hasAttribute;
100 }
101
102 void doSetName(const Name &name) override {
103 nameAttribute_.add(name);
104 }
105
106 const Name &name() const override {
107 return nameAttribute_.get();
108 }
109
110 void doSetDoc(const std::string &doc) override {
111 docAttribute_.add(doc);
112 }
113
114 const std::string &getDoc() const override {
115 return docAttribute_.get();
116 }
117
118 void doAddLeaf(const NodePtr &newLeaf) final {
119 leafAttributes_.add(newLeaf);
120 }
121
122 size_t leaves() const override {
123 return leafAttributes_.size();
124 }
125
126 const NodePtr &leafAt(size_t index) const override {
127 return leafAttributes_.get(index);
128 }
129
130 void doAddName(const std::string &name) override {
131 if (!nameIndex_.add(name, leafNameAttributes_.size())) {
132 throw Exception(boost::format("Cannot add duplicate name: %1%") % name);
133 }
134 leafNameAttributes_.add(name);
135 }
136
137 size_t names() const override {
138 return leafNameAttributes_.size();
139 }
140
141 const std::string &nameAt(size_t index) const override {
142 return leafNameAttributes_.get(index);
143 }
144
145 bool nameIndex(const std::string &name, size_t &index) const override {
146 return nameIndex_.lookup(name, index);
147 }
148
149 void doSetFixedSize(size_t size) override {
150 sizeAttribute_.add(size);
151 }
152
153 size_t fixedSize() const override {
154 return sizeAttribute_.get();
155 }
156
157 bool isValid() const override = 0;
158
159 void printBasicInfo(std::ostream &os) const override;
160
161 void setLeafToSymbolic(size_t index, const NodePtr &node) override;
162
163 void doAddCustomAttribute(const CustomAttributes &customAttributes) override {
164 customAttributes_.add(customAttributes);
165 }
166
167 SchemaResolution furtherResolution(const Node &reader) const {
168 SchemaResolution match = RESOLVE_NO_MATCH;
169
170 if (reader.type() == AVRO_SYMBOLIC) {
171
172 // resolve the symbolic type, and check again
173 const NodePtr &node = reader.leafAt(0);
174 match = resolve(*node);
175 } else if (reader.type() == AVRO_UNION) {
176
177 // in this case, need to see if there is an exact match for the
178 // writer's type, or if not, the first one that can be promoted to a
179 // match
180
181 for (size_t i = 0; i < reader.leaves(); ++i) {
182
183 const NodePtr &node = reader.leafAt(i);
184 SchemaResolution thisMatch = resolve(*node);
185
186 // if matched then the search is done
187 if (thisMatch == RESOLVE_MATCH) {
188 match = thisMatch;
189 break;
190 }
191
192 // thisMatch is either no match, or promotable, this will set match to
193 // promotable if it hasn't been set already
194 if (match == RESOLVE_NO_MATCH) {
195 match = thisMatch;
196 }
197 }
198 }
199
200 return match;
201 }
202
203 NameConcept nameAttribute_;
204
205 // Rem: NameConcept type is HasName (= SingleAttribute<Name>), we use std::string instead
206 concepts::SingleAttribute<std::string> docAttribute_;
208 LeavesConcept leafAttributes_;
209 LeafNamesConcept leafNameAttributes_;
210 MultiAttributesConcept customAttributes_;
211 SizeConcept sizeAttribute_;
212 concepts::NameIndexConcept<LeafNamesConcept> nameIndex_;
213 };
214
215 using NoName = concepts::NoAttribute<Name>;
216 using HasName = concepts::SingleAttribute<Name>;
217
218 using HasDoc = concepts::SingleAttribute<std::string>;
219
220 using NoLeaves = concepts::NoAttribute<NodePtr>;
221 using SingleLeaf = concepts::SingleAttribute<NodePtr>;
222 using MultiLeaves = concepts::MultiAttribute<NodePtr>;
223
224 using NoLeafNames = concepts::NoAttribute<std::string>;
225 using LeafNames = concepts::MultiAttribute<std::string>;
226 using MultiAttributes = concepts::MultiAttribute<CustomAttributes>;
227 using NoAttributes = concepts::NoAttribute<CustomAttributes>;
228
229 using NoSize = concepts::NoAttribute<int>;
230 using HasSize = concepts::SingleAttribute<int>;
231
232 using NodeImplPrimitive = NodeImpl<NoName, NoLeaves, NoLeafNames, MultiAttributes, NoSize>;
233 using NodeImplSymbolic = NodeImpl<HasName, NoLeaves, NoLeafNames, NoAttributes, NoSize>;
234
235 using NodeImplRecord = NodeImpl<HasName, MultiLeaves, LeafNames, MultiAttributes, NoSize>;
236 using NodeImplEnum = NodeImpl<HasName, NoLeaves, LeafNames, NoAttributes, NoSize>;
237 using NodeImplArray = NodeImpl<NoName, SingleLeaf, NoLeafNames, NoAttributes, NoSize>;
238 using NodeImplMap = NodeImpl<NoName, MultiLeaves, NoLeafNames, NoAttributes, NoSize>;
239 using NodeImplUnion = NodeImpl<NoName, MultiLeaves, NoLeafNames, NoAttributes, NoSize>;
240 using NodeImplFixed = NodeImpl<HasName, NoLeaves, NoLeafNames, NoAttributes, HasSize>;
241
242 class AVRO_DECL NodePrimitive : public NodeImplPrimitive {
243 public:
244 explicit NodePrimitive(Type type) : NodeImplPrimitive(type) {}
245
246 SchemaResolution resolve(const Node &reader) const override;
247
248 void printJson(std::ostream &os, size_t depth) const override;
249
250 bool isValid() const override {
251 return true;
252 }
253
254 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
255 };
256
257 class AVRO_DECL NodeSymbolic : public NodeImplSymbolic {
258 using NodeWeakPtr = std::weak_ptr<Node>;
259
260 public:
261 NodeSymbolic() : NodeImplSymbolic(AVRO_SYMBOLIC) {}
262
263 explicit NodeSymbolic(const HasName &name) : NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoAttributes(), NoSize()) {}
264
265 NodeSymbolic(const HasName &name, const NodePtr &n) : NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoAttributes(), NoSize()), actualNode_(n) {}
266 SchemaResolution resolve(const Node &reader) const override;
267
268 void printJson(std::ostream &os, size_t depth) const override;
269
270 bool isValid() const override {
271 return (nameAttribute_.size() == 1);
272 }
273
274 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
275
276 bool isSet() const {
277 return (actualNode_.lock() != nullptr);
278 }
279
280 NodePtr getNode() const {
281 NodePtr node = actualNode_.lock();
282 if (!node) {
283 throw Exception(boost::format("Could not follow symbol %1%") % name());
284 }
285 return node;
286 }
287
288 void setNode(const NodePtr &node) {
289 actualNode_ = node;
290 }
291
292 protected:
293 NodeWeakPtr actualNode_;
294 };
295
296 class AVRO_DECL NodeRecord : public NodeImplRecord {
297 std::vector<GenericDatum> defaultValues;
298
299 public:
300 NodeRecord() : NodeImplRecord(AVRO_RECORD) {}
301 NodeRecord(const HasName &name, const MultiLeaves &fields,
302 const LeafNames &fieldsNames,
303 std::vector<GenericDatum> dv);
304
305 NodeRecord(const HasName &name, const HasDoc &doc, const MultiLeaves &fields,
306 const LeafNames &fieldsNames,
307 std::vector<GenericDatum> dv) : NodeImplRecord(AVRO_RECORD, name, doc, fields, fieldsNames, MultiAttributes(), NoSize()),
308 defaultValues(std::move(dv)) {
309 leafNameCheck();
310 }
311
312 NodeRecord(const HasName &name, const MultiLeaves &fields,
313 const LeafNames &fieldsNames,
314 const std::vector<GenericDatum>& dv,
315 const MultiAttributes &customAttributes) :
316 NodeImplRecord(AVRO_RECORD, name, fields, fieldsNames, customAttributes, NoSize()),
317 defaultValues(dv) {
318 leafNameCheck();
319 }
320
321 NodeRecord(const HasName &name, const HasDoc &doc, const MultiLeaves &fields,
322 const LeafNames &fieldsNames,
323 const std::vector<GenericDatum>& dv,
324 const MultiAttributes &customAttributes) :
325 NodeImplRecord(AVRO_RECORD, name, doc, fields, fieldsNames, customAttributes, NoSize()),
326 defaultValues(dv) {
327 leafNameCheck();
328 }
329
330 void swap(NodeRecord &r) {
331 NodeImplRecord::swap(r);
332 defaultValues.swap(r.defaultValues);
333 }
334
335 SchemaResolution resolve(const Node &reader) const override;
336
337 void printJson(std::ostream &os, size_t depth) const override;
338
339 bool isValid() const override {
340 return ((nameAttribute_.size() == 1) &&
341 (leafAttributes_.size() == leafNameAttributes_.size()) &&
342 (customAttributes_.size() == 0 ||
343 customAttributes_.size() == leafAttributes_.size()));
344 }
345
346 const GenericDatum &defaultValueAt(size_t index) override {
347 return defaultValues[index];
348 }
349
350 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
351
352 private:
353 // check if leaf name is valid Name and is not duplicate
354 void leafNameCheck() {
355 for (size_t i = 0; i < leafNameAttributes_.size(); ++i) {
356 if (!nameIndex_.add(leafNameAttributes_.get(i), i)) {
357 throw Exception(boost::format(
358 "Cannot add duplicate field: %1%")
359 % leafNameAttributes_.get(i));
360 }
361 }
362 }
363 };
364
365 class AVRO_DECL NodeEnum : public NodeImplEnum {
366 public:
367 NodeEnum() : NodeImplEnum(AVRO_ENUM) {}
368
369 NodeEnum(const HasName &name, const LeafNames &symbols) : NodeImplEnum(AVRO_ENUM, name, NoLeaves(), symbols, NoAttributes(), NoSize()) {
370 for (size_t i = 0; i < leafNameAttributes_.size(); ++i) {
371 if (!nameIndex_.add(leafNameAttributes_.get(i), i)) {
372 throw Exception(boost::format("Cannot add duplicate enum: %1%") % leafNameAttributes_.get(i));
373 }
374 }
375 }
376
377 SchemaResolution resolve(const Node &reader) const override;
378
379 void printJson(std::ostream &os, size_t depth) const override;
380
381 bool isValid() const override {
382 return (
383 (nameAttribute_.size() == 1) && (leafNameAttributes_.size() > 0));
384 }
385
386 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
387 };
388
389 class AVRO_DECL NodeArray : public NodeImplArray {
390 public:
391 NodeArray() : NodeImplArray(AVRO_ARRAY) {}
392
393 explicit NodeArray(const SingleLeaf &items) : NodeImplArray(AVRO_ARRAY, NoName(), items, NoLeafNames(), NoAttributes(), NoSize()) {}
394
395 SchemaResolution resolve(const Node &reader) const override;
396
397 void printJson(std::ostream &os, size_t depth) const override;
398
399 bool isValid() const override {
400 return (leafAttributes_.size() == 1);
401 }
402
403 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
404 };
405
406 class AVRO_DECL NodeMap : public NodeImplMap {
407 public:
408 NodeMap();
409
410 explicit NodeMap(const SingleLeaf &values) : NodeImplMap(AVRO_MAP, NoName(), MultiLeaves(values), NoLeafNames(), NoAttributes(), NoSize()) {
411 // need to add the key for the map too
412 NodePtr key(new NodePrimitive(AVRO_STRING));
413 doAddLeaf(key);
414
415 // key goes before value
416 std::swap(leafAttributes_.get(0), leafAttributes_.get(1));
417 }
418
419 SchemaResolution resolve(const Node &reader) const override;
420
421 void printJson(std::ostream &os, size_t depth) const override;
422
423 bool isValid() const override {
424 return (leafAttributes_.size() == 2);
425 }
426
427 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
428 };
429
430 class AVRO_DECL NodeUnion : public NodeImplUnion {
431 public:
432 NodeUnion() : NodeImplUnion(AVRO_UNION) {}
433
434 explicit NodeUnion(const MultiLeaves &types) : NodeImplUnion(AVRO_UNION, NoName(), types, NoLeafNames(), NoAttributes(), NoSize()) {}
435
436 SchemaResolution resolve(const Node &reader) const override;
437
438 void printJson(std::ostream &os, size_t depth) const override;
439
440 bool isValid() const override {
441 std::set<std::string> seen;
442 if (leafAttributes_.size() >= 1) {
443 for (size_t i = 0; i < leafAttributes_.size(); ++i) {
444 std::string name;
445 const NodePtr &n = leafAttributes_.get(i);
446 switch (n->type()) {
447 case AVRO_STRING:
448 name = "string";
449 break;
450 case AVRO_BYTES:
451 name = "bytes";
452 break;
453 case AVRO_INT:
454 name = "int";
455 break;
456 case AVRO_LONG:
457 name = "long";
458 break;
459 case AVRO_FLOAT:
460 name = "float";
461 break;
462 case AVRO_DOUBLE:
463 name = "double";
464 break;
465 case AVRO_BOOL:
466 name = "bool";
467 break;
468 case AVRO_NULL:
469 name = "null";
470 break;
471 case AVRO_ARRAY:
472 name = "array";
473 break;
474 case AVRO_MAP:
475 name = "map";
476 break;
477 case AVRO_RECORD:
478 case AVRO_ENUM:
479 case AVRO_UNION:
480 case AVRO_FIXED:
481 case AVRO_SYMBOLIC:
482 name = n->name().fullname();
483 break;
484 default: return false;
485 }
486 if (seen.find(name) != seen.end()) {
487 return false;
488 }
489 seen.insert(name);
490 }
491 return true;
492 }
493 return false;
494 }
495
496 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
497 };
498
499 class AVRO_DECL NodeFixed : public NodeImplFixed {
500 public:
501 NodeFixed() : NodeImplFixed(AVRO_FIXED) {}
502
503 NodeFixed(const HasName &name, const HasSize &size) : NodeImplFixed(AVRO_FIXED, name, NoLeaves(), NoLeafNames(), NoAttributes(), size) {}
504
505 SchemaResolution resolve(const Node &reader) const override;
506
507 void printJson(std::ostream &os, size_t depth) const override;
508
509 bool isValid() const override {
510 return (
511 (nameAttribute_.size() == 1) && (sizeAttribute_.size() == 1));
512 }
513
514 void printDefaultToJson(const GenericDatum &g, std::ostream &os, size_t depth) const override;
515 };
516
517 template<class A, class B, class C, class D, class E>
518 inline void
519 NodeImpl<A, B, C, D, E>::setLeafToSymbolic(size_t index, const NodePtr &node) {
520 if (!B::hasAttribute) {
521 throw Exception("Cannot change leaf node for nonexistent leaf");
522 }
523
524 auto &replaceNode = const_cast<NodePtr &>(leafAttributes_.get(index));
525 if (replaceNode->name() != node->name()) {
526 throw Exception("Symbolic name does not match the name of the schema it references");
527 }
528
529 auto symbol = std::make_shared<NodeSymbolic>();
530 symbol->setName(node->name());
531 symbol->setNode(node);
532 replaceNode = symbol;
533 }
534
535 template<class A, class B, class C, class D, class E>
536 inline void
537 NodeImpl<A, B, C, D, E>::printBasicInfo(std::ostream &os) const {
538 os << type();
539 if (hasName()) {
540 os << ' ' << nameAttribute_.get();
541 }
542
543 if (E::hasAttribute) {
544 os << " " << sizeAttribute_.get();
545 }
546 os << '\n';
547 size_t count = leaves();
548 count = count ? count : names();
549 for (size_t i = 0; i < count; ++i) {
550 if (C::hasAttribute) {
551 os << "name " << nameAt(i) << '\n';
552 }
553 if (type() != AVRO_SYMBOLIC && leafAttributes_.hasAttribute) {
554 leafAt(i)->printBasicInfo(os);
555 }
556 }
557 if (isCompound(type())) {
558 os << "end " << type() << '\n';
559 }
560 }
561
562 inline NodePtr resolveSymbol(const NodePtr &node) {
563 if (node->type() != AVRO_SYMBOLIC) {
564 throw Exception("Only symbolic nodes may be resolved");
565 }
566 std::shared_ptr<NodeSymbolic> symNode = std::static_pointer_cast<NodeSymbolic>(node);
567 return symNode->getNode();
568 }
569
570 template<typename T>
571 inline std::string intToHex(T i) {
572 std::stringstream stream;
573 stream << "\\u"
574 << std::setfill('0') << std::setw(sizeof(T))
575 << std::hex << i;
576 return stream.str();
577 }
578
579 } // namespace avro
580
581 #endif
avro::AVRO_NULL
@ AVRO_NULL
Definition: Types.hh:40
avro::Node
Node is the building block for parse trees.
Definition: Node.hh:91
avro::NodeFixed
Definition: NodeImpl.hh:499
avro::AVRO_LONG
@ AVRO_LONG
Definition: Types.hh:36
avro::AVRO_ENUM
@ AVRO_ENUM
Definition: Types.hh:43
avro::NodeMap
Definition: NodeImpl.hh:406
avro::RESOLVE_MATCH
@ RESOLVE_MATCH
The schemas match at a cursory level.
Definition: SchemaResolution.hh:37
avro::AVRO_FLOAT
@ AVRO_FLOAT
Definition: Types.hh:37
avro::AVRO_BOOL
@ AVRO_BOOL
Definition: Types.hh:39
avro::AVRO_STRING
@ AVRO_STRING
Definition: Types.hh:33
avro::Name
Definition: Node.hh:42
avro::NodeImpl::leafAttributes_
LeavesConcept leafAttributes_
Doc used to compare schemas.
Definition: NodeImpl.hh:208
avro::NodeEnum
Definition: NodeImpl.hh:365
avro::AVRO_BYTES
@ AVRO_BYTES
Definition: Types.hh:34
avro::NodeRecord
Definition: NodeImpl.hh:296
avro::concepts::MultiAttribute
Definition: NodeConcepts.hh:121
avro::NodeSymbolic
Definition: NodeImpl.hh:257
avro::CustomAttributes
Definition: CustomAttributes.hh:33
avro::GenericDatum
Generic datum which can hold any Avro type.
Definition: GenericDatum.hh:61
avro::concepts::NoAttribute
Definition: NodeConcepts.hh:50
avro::NodePrimitive
Definition: NodeImpl.hh:242
avro::AVRO_INT
@ AVRO_INT
Definition: Types.hh:35
avro
A bunch of templates and specializations for encoding and decoding specific types.
Definition: AvroParse.hh:30
avro::NodeUnion
Definition: NodeImpl.hh:430
avro::RESOLVE_NO_MATCH
@ RESOLVE_NO_MATCH
The schemas definitely do not match.
Definition: SchemaResolution.hh:30
avro::AVRO_ARRAY
@ AVRO_ARRAY
Definition: Types.hh:44
avro::AVRO_DOUBLE
@ AVRO_DOUBLE
Definition: Types.hh:38
avro::isCompound
constexpr bool isCompound(Type t) noexcept
Returns true if and only if the given type is a non primitive valid type.
Definition: Types.hh:71
avro::AVRO_FIXED
@ AVRO_FIXED
Definition: Types.hh:47
avro::SchemaResolution
SchemaResolution
Definition: SchemaResolution.hh:26
avro::concepts::SingleAttribute< std::string >
avro::AVRO_RECORD
@ AVRO_RECORD
Definition: Types.hh:42
avro::NodeImpl
Implementation details for Node.
Definition: NodeImpl.hh:48
avro::concepts::NameIndexConcept< LeafNamesConcept >
avro::AVRO_SYMBOLIC
@ AVRO_SYMBOLIC
Definition: Types.hh:53
avro::NodeArray
Definition: NodeImpl.hh:389
avro::Type
Type
The "type" for the schema.
Definition: Types.hh:31
avro::AVRO_UNION
@ AVRO_UNION
Definition: Types.hh:46
avro::AVRO_MAP
@ AVRO_MAP
Definition: Types.hh:45
avro::Exception
Wrapper for std::runtime_error that provides convenience constructor for boost::format objects.
Definition: Exception.hh:31