Revision 48434
Added by Marek Horst over 7 years ago
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/pom.xml | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> |
|
3 |
<parent> |
|
4 |
<groupId>eu.dnetlib</groupId> |
|
5 |
<artifactId>dnet45-parent</artifactId> |
|
6 |
<version>1.0.0</version> |
|
7 |
</parent> |
|
8 |
<modelVersion>4.0.0</modelVersion> |
|
9 |
<artifactId>icm-iis-3rdparty-avrojsoncoders</artifactId> |
|
10 |
<packaging>jar</packaging> |
|
11 |
<version>1.0.3</version> |
|
12 |
|
|
13 |
<scm> |
|
14 |
<developerConnection> |
|
15 |
scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3 |
|
16 |
</developerConnection> |
|
17 |
</scm> |
|
18 |
|
|
19 |
<properties> |
|
20 |
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
|
21 |
</properties> |
|
22 |
<dependencies> |
|
23 |
<dependency> |
|
24 |
<groupId>junit</groupId> |
|
25 |
<artifactId>junit</artifactId> |
|
26 |
<version>4.10</version> |
|
27 |
<scope>test</scope> |
|
28 |
</dependency> |
|
29 |
<dependency> |
|
30 |
<groupId>org.apache.avro</groupId> |
|
31 |
<artifactId>avro</artifactId> |
|
32 |
<version>${iis.avro.version}</version> |
|
33 |
</dependency> |
|
34 |
</dependencies> |
|
35 |
<build> |
|
36 |
<plugins> |
|
37 |
<!-- Plugin that generates Java classes from Avro schemas --> |
|
38 |
<plugin> |
|
39 |
<groupId>org.apache.avro</groupId> |
|
40 |
<artifactId>avro-maven-plugin</artifactId> |
|
41 |
<version>${iis.avro.version}</version> |
|
42 |
<executions> |
|
43 |
<execution> |
|
44 |
<phase>generate-sources</phase> |
|
45 |
<goals> |
|
46 |
<goal>schema</goal> |
|
47 |
<goal>idl-protocol</goal> |
|
48 |
</goals> |
|
49 |
<configuration> |
|
50 |
<sourceDirectory>${project.basedir}/src/test/resources/</sourceDirectory> |
|
51 |
<outputDirectory>${project.basedir}/target/generated-sources/java/</outputDirectory> |
|
52 |
</configuration> |
|
53 |
</execution> |
|
54 |
</executions> |
|
55 |
</plugin> |
|
56 |
<!-- This plugin makes the Maven->Update Project Configuration not forget |
|
57 |
about the "target/generated-sources/java" source path --> |
|
58 |
<plugin> |
|
59 |
<groupId>org.codehaus.mojo</groupId> |
|
60 |
<artifactId>build-helper-maven-plugin</artifactId> |
|
61 |
<executions> |
|
62 |
<execution> |
|
63 |
<id>add-source</id> |
|
64 |
<phase>generate-sources</phase> |
|
65 |
<goals> |
|
66 |
<goal>add-source</goal> |
|
67 |
</goals> |
|
68 |
<configuration> |
|
69 |
<sources> |
|
70 |
<source>${project.build.directory}/generated-sources/java/</source> |
|
71 |
</sources> |
|
72 |
</configuration> |
|
73 |
</execution> |
|
74 |
</executions> |
|
75 |
</plugin> |
|
76 |
</plugins> |
|
77 |
<pluginManagement> |
|
78 |
<plugins> |
|
79 |
<!--This plugin's configuration is used to store Eclipse m2e settings |
|
80 |
only. It has no influence on the Maven build itself. --> |
|
81 |
<plugin> |
|
82 |
<groupId>org.eclipse.m2e</groupId> |
|
83 |
<artifactId>lifecycle-mapping</artifactId> |
|
84 |
<version>1.0.0</version> |
|
85 |
<configuration> |
|
86 |
<lifecycleMappingMetadata> |
|
87 |
<pluginExecutions> |
|
88 |
<pluginExecution> |
|
89 |
<pluginExecutionFilter> |
|
90 |
<groupId> |
|
91 |
org.apache.avro |
|
92 |
</groupId> |
|
93 |
<artifactId> |
|
94 |
avro-maven-plugin |
|
95 |
</artifactId> |
|
96 |
<versionRange> |
|
97 |
[1.7.4,) |
|
98 |
</versionRange> |
|
99 |
<goals> |
|
100 |
<goal>idl-protocol</goal> |
|
101 |
<goal>schema</goal> |
|
102 |
</goals> |
|
103 |
</pluginExecutionFilter> |
|
104 |
<action> |
|
105 |
<ignore /> |
|
106 |
</action> |
|
107 |
</pluginExecution> |
|
108 |
<pluginExecution> |
|
109 |
<pluginExecutionFilter> |
|
110 |
<groupId> |
|
111 |
org.codehaus.mojo |
|
112 |
</groupId> |
|
113 |
<artifactId> |
|
114 |
build-helper-maven-plugin |
|
115 |
</artifactId> |
|
116 |
<versionRange> |
|
117 |
[1.7,) |
|
118 |
</versionRange> |
|
119 |
<goals> |
|
120 |
<goal>add-source</goal> |
|
121 |
</goals> |
|
122 |
</pluginExecutionFilter> |
|
123 |
<action> |
|
124 |
<ignore /> |
|
125 |
</action> |
|
126 |
</pluginExecution> |
|
127 |
</pluginExecutions> |
|
128 |
</lifecycleMappingMetadata> |
|
129 |
</configuration> |
|
130 |
</plugin> |
|
131 |
</plugins> |
|
132 |
</pluginManagement> |
|
133 |
</build> |
|
134 |
<repositories> |
|
135 |
</repositories> |
|
136 |
</project> |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/main/java/org/apache/avro/io/package-info.java | ||
---|---|---|
1 |
/** |
|
2 |
* Provides hacked version of classes used to convert between Avro and JSON |
|
3 |
* formats. The hacked version allows for not defining explicitly types of |
|
4 |
* fields in a union when one of the union-ed fields is a null. |
|
5 |
* |
|
6 |
* Generally the problem with the original implementation is in the way |
|
7 |
* the Avro library converts JSON to Avro file when you use the schema with |
|
8 |
* a "union" field (and we use this kind of field a lot to make a certain |
|
9 |
* field null-able (e.g. "union {null, int} age")). |
|
10 |
* |
|
11 |
* For example when you use the schema (Avro IDL format): |
|
12 |
* |
|
13 |
* record Document{ |
|
14 |
* int id; |
|
15 |
* union {null, string} title=null; |
|
16 |
* array<int> authorIds; |
|
17 |
* } |
|
18 |
* |
|
19 |
* The JSON that Avro expects must have the type of the union field |
|
20 |
* defined explicitly: |
|
21 |
* |
|
22 |
* {"authorIds": [1, 20], "id": 20, "title": {"string": "An extraordinary book"}} |
|
23 |
* {"authorIds": [10, 6, 1], "id": 1, "title": {"string": "Basics of the basics"}} |
|
24 |
* {"authorIds": [], "id": 2, "title": null} |
|
25 |
* {"authorIds": [1, 6], "id": 6, "title": {"string": "Even more of interesting stuff"}} |
|
26 |
* |
|
27 |
* One way of dealing with it is just enforcing production of JSON conforming |
|
28 |
* to this convention by our modules. However such an approach seems to |
|
29 |
* introduce an unneeded dose of complexity to the produced JSON, |
|
30 |
* so I decided to modify the Avro library to make it accept JSON without the |
|
31 |
* type in the union field defined explicitly. |
|
32 |
* |
|
33 |
* This convention of defining JSON structure is necessary in the general case |
|
34 |
* (see http://mail-archives.apache.org/mod_mbox/avro-user/201206.mbox/%3CCALEq1Z84=20PfJahT7DKtXEAGPFfOoHtVYXiyi+O+cZYGdXBNQ@mail.gmail.com%3E), |
|
35 |
* but in our case of using "union" only to mark a field nullable |
|
36 |
* it does not seem to be needed. |
|
37 |
*/ |
|
38 |
package org.apache.avro.io; |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/main/java/org/apache/avro/io/HackedJsonDecoder.java | ||
---|---|---|
1 |
/** |
|
2 |
* Licensed to the Apache Software Foundation (ASF) under one |
|
3 |
* or more contributor license agreements. See the NOTICE file |
|
4 |
* distributed with this work for additional information |
|
5 |
* regarding copyright ownership. The ASF licenses this file |
|
6 |
* to you under the Apache License, Version 2.0 (the |
|
7 |
* "License"); you may not use this file except in compliance |
|
8 |
* with the License. You may obtain a copy of the License at |
|
9 |
* |
|
10 |
* http://www.apache.org/licenses/LICENSE-2.0 |
|
11 |
* |
|
12 |
* Unless required by applicable law or agreed to in writing, software |
|
13 |
* distributed under the License is distributed on an "AS IS" BASIS, |
|
14 |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
15 |
* See the License for the specific language governing permissions and |
|
16 |
* limitations under the License. |
|
17 |
*/ |
|
18 |
package org.apache.avro.io; |
|
19 |
|
|
20 |
import java.io.EOFException; |
|
21 |
import java.io.IOException; |
|
22 |
import java.io.InputStream; |
|
23 |
import java.math.BigDecimal; |
|
24 |
import java.math.BigInteger; |
|
25 |
import java.nio.ByteBuffer; |
|
26 |
import java.util.ArrayList; |
|
27 |
import java.util.HashMap; |
|
28 |
import java.util.List; |
|
29 |
import java.util.Map; |
|
30 |
import java.util.Stack; |
|
31 |
|
|
32 |
import org.apache.avro.AvroTypeException; |
|
33 |
import org.apache.avro.Schema; |
|
34 |
import org.apache.avro.io.parsing.JsonGrammarGenerator; |
|
35 |
import org.apache.avro.io.parsing.Parser; |
|
36 |
import org.apache.avro.io.parsing.Symbol; |
|
37 |
import org.apache.avro.util.Utf8; |
|
38 |
import org.codehaus.jackson.Base64Variant; |
|
39 |
import org.codehaus.jackson.JsonFactory; |
|
40 |
import org.codehaus.jackson.JsonLocation; |
|
41 |
import org.codehaus.jackson.JsonParser; |
|
42 |
import org.codehaus.jackson.JsonStreamContext; |
|
43 |
import org.codehaus.jackson.JsonToken; |
|
44 |
import org.codehaus.jackson.ObjectCodec; |
|
45 |
|
|
46 |
/** A {@link Decoder} for Avro's JSON data encoding. |
|
47 |
* </p> |
|
48 |
* Construct using {@link DecoderFactory}. |
|
49 |
* </p> |
|
50 |
* JsonDecoder is not thread-safe. |
|
51 |
* */ |
|
52 |
public class HackedJsonDecoder extends ParsingDecoder |
|
53 |
implements Parser.ActionHandler { |
|
54 |
|
|
55 |
private JsonParser in; |
|
56 |
private static JsonFactory jsonFactory = new JsonFactory(); |
|
57 |
Stack<ReorderBuffer> reorderBuffers = new Stack<ReorderBuffer>(); |
|
58 |
ReorderBuffer currentReorderBuffer; |
|
59 |
|
|
60 |
private static class ReorderBuffer { |
|
61 |
public Map<String, List<JsonElement>> savedFields = new HashMap<String, List<JsonElement>>(); |
|
62 |
public JsonParser origParser = null; |
|
63 |
} |
|
64 |
|
|
65 |
static final String CHARSET = "ISO-8859-1"; |
|
66 |
|
|
67 |
private HackedJsonDecoder(Symbol root, InputStream in) throws IOException { |
|
68 |
super(root); |
|
69 |
configure(in); |
|
70 |
} |
|
71 |
|
|
72 |
private HackedJsonDecoder(Symbol root, String in) throws IOException { |
|
73 |
super(root); |
|
74 |
configure(in); |
|
75 |
} |
|
76 |
|
|
77 |
public HackedJsonDecoder(Schema schema, InputStream in) throws IOException { |
|
78 |
this(getSymbol(schema), in); |
|
79 |
} |
|
80 |
|
|
81 |
public HackedJsonDecoder(Schema schema, String in) throws IOException { |
|
82 |
this(getSymbol(schema), in); |
|
83 |
} |
|
84 |
|
|
85 |
private static Symbol getSymbol(Schema schema) { |
|
86 |
if (null == schema) { |
|
87 |
throw new NullPointerException("Schema cannot be null!"); |
|
88 |
} |
|
89 |
return new JsonGrammarGenerator().generate(schema); |
|
90 |
} |
|
91 |
|
|
92 |
/** |
|
93 |
* Reconfigures this JsonDecoder to use the InputStream provided. |
|
94 |
* <p/> |
|
95 |
* If the InputStream provided is null, a NullPointerException is thrown. |
|
96 |
* <p/> |
|
97 |
* Otherwise, this JsonDecoder will reset its state and then |
|
98 |
* reconfigure its input. |
|
99 |
* @param in |
|
100 |
* The IntputStream to read from. Cannot be null. |
|
101 |
* @throws IOException |
|
102 |
* @return this JsonDecoder |
|
103 |
*/ |
|
104 |
public HackedJsonDecoder configure(InputStream in) throws IOException { |
|
105 |
if (null == in) { |
|
106 |
throw new NullPointerException("InputStream to read from cannot be null!"); |
|
107 |
} |
|
108 |
parser.reset(); |
|
109 |
this.in = jsonFactory.createJsonParser(in); |
|
110 |
this.in.nextToken(); |
|
111 |
return this; |
|
112 |
} |
|
113 |
|
|
114 |
/** |
|
115 |
* Reconfigures this JsonDecoder to use the String provided for input. |
|
116 |
* <p/> |
|
117 |
* If the String provided is null, a NullPointerException is thrown. |
|
118 |
* <p/> |
|
119 |
* Otherwise, this JsonDecoder will reset its state and then |
|
120 |
* reconfigure its input. |
|
121 |
* @param in |
|
122 |
* The String to read from. Cannot be null. |
|
123 |
* @throws IOException |
|
124 |
* @return this JsonDecoder |
|
125 |
*/ |
|
126 |
public HackedJsonDecoder configure(String in) throws IOException { |
|
127 |
if (null == in) { |
|
128 |
throw new NullPointerException("String to read from cannot be null!"); |
|
129 |
} |
|
130 |
parser.reset(); |
|
131 |
this.in = new JsonFactory().createJsonParser(in); |
|
132 |
this.in.nextToken(); |
|
133 |
return this; |
|
134 |
} |
|
135 |
|
|
136 |
private void advance(Symbol symbol) throws IOException { |
|
137 |
this.parser.processTrailingImplicitActions(); |
|
138 |
if (in.getCurrentToken() == null && this.parser.depth() == 1) |
|
139 |
throw new EOFException(); |
|
140 |
parser.advance(symbol); |
|
141 |
} |
|
142 |
|
|
143 |
@Override |
|
144 |
public void readNull() throws IOException { |
|
145 |
advance(Symbol.NULL); |
|
146 |
if (in.getCurrentToken() == JsonToken.VALUE_NULL) { |
|
147 |
in.nextToken(); |
|
148 |
} else { |
|
149 |
throw error("null"); |
|
150 |
} |
|
151 |
} |
|
152 |
|
|
153 |
@Override |
|
154 |
public boolean readBoolean() throws IOException { |
|
155 |
advance(Symbol.BOOLEAN); |
|
156 |
JsonToken t = in.getCurrentToken(); |
|
157 |
if (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE) { |
|
158 |
in.nextToken(); |
|
159 |
return t == JsonToken.VALUE_TRUE; |
|
160 |
} else { |
|
161 |
throw error("boolean"); |
|
162 |
} |
|
163 |
} |
|
164 |
|
|
165 |
@Override |
|
166 |
public int readInt() throws IOException { |
|
167 |
advance(Symbol.INT); |
|
168 |
if (in.getCurrentToken().isNumeric()) { |
|
169 |
int result = in.getIntValue(); |
|
170 |
in.nextToken(); |
|
171 |
return result; |
|
172 |
} else { |
|
173 |
throw error("int"); |
|
174 |
} |
|
175 |
} |
|
176 |
|
|
177 |
@Override |
|
178 |
public long readLong() throws IOException { |
|
179 |
advance(Symbol.LONG); |
|
180 |
if (in.getCurrentToken().isNumeric()) { |
|
181 |
long result = in.getLongValue(); |
|
182 |
in.nextToken(); |
|
183 |
return result; |
|
184 |
} else { |
|
185 |
throw error("long"); |
|
186 |
} |
|
187 |
} |
|
188 |
|
|
189 |
@Override |
|
190 |
public float readFloat() throws IOException { |
|
191 |
advance(Symbol.FLOAT); |
|
192 |
if (in.getCurrentToken().isNumeric()) { |
|
193 |
float result = in.getFloatValue(); |
|
194 |
in.nextToken(); |
|
195 |
return result; |
|
196 |
} else { |
|
197 |
throw error("float"); |
|
198 |
} |
|
199 |
} |
|
200 |
|
|
201 |
@Override |
|
202 |
public double readDouble() throws IOException { |
|
203 |
advance(Symbol.DOUBLE); |
|
204 |
if (in.getCurrentToken().isNumeric()) { |
|
205 |
double result = in.getDoubleValue(); |
|
206 |
in.nextToken(); |
|
207 |
return result; |
|
208 |
} else { |
|
209 |
throw error("double"); |
|
210 |
} |
|
211 |
} |
|
212 |
|
|
213 |
@Override |
|
214 |
public Utf8 readString(Utf8 old) throws IOException { |
|
215 |
return new Utf8(readString()); |
|
216 |
} |
|
217 |
|
|
218 |
@Override |
|
219 |
public String readString() throws IOException { |
|
220 |
advance(Symbol.STRING); |
|
221 |
if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) { |
|
222 |
parser.advance(Symbol.MAP_KEY_MARKER); |
|
223 |
if (in.getCurrentToken() != JsonToken.FIELD_NAME) { |
|
224 |
throw error("map-key"); |
|
225 |
} |
|
226 |
} else { |
|
227 |
if (in.getCurrentToken() != JsonToken.VALUE_STRING) { |
|
228 |
throw error("string"); |
|
229 |
} |
|
230 |
} |
|
231 |
String result = in.getText(); |
|
232 |
in.nextToken(); |
|
233 |
return result; |
|
234 |
} |
|
235 |
|
|
236 |
@Override |
|
237 |
public void skipString() throws IOException { |
|
238 |
advance(Symbol.STRING); |
|
239 |
if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) { |
|
240 |
parser.advance(Symbol.MAP_KEY_MARKER); |
|
241 |
if (in.getCurrentToken() != JsonToken.FIELD_NAME) { |
|
242 |
throw error("map-key"); |
|
243 |
} |
|
244 |
} else { |
|
245 |
if (in.getCurrentToken() != JsonToken.VALUE_STRING) { |
|
246 |
throw error("string"); |
|
247 |
} |
|
248 |
} |
|
249 |
in.nextToken(); |
|
250 |
} |
|
251 |
|
|
252 |
@Override |
|
253 |
public ByteBuffer readBytes(ByteBuffer old) throws IOException { |
|
254 |
advance(Symbol.BYTES); |
|
255 |
if (in.getCurrentToken() == JsonToken.VALUE_STRING) { |
|
256 |
byte[] result = readByteArray(); |
|
257 |
in.nextToken(); |
|
258 |
return ByteBuffer.wrap(result); |
|
259 |
} else { |
|
260 |
throw error("bytes"); |
|
261 |
} |
|
262 |
} |
|
263 |
|
|
264 |
private byte[] readByteArray() throws IOException { |
|
265 |
byte[] result = in.getText().getBytes(CHARSET); |
|
266 |
return result; |
|
267 |
} |
|
268 |
|
|
269 |
@Override |
|
270 |
public void skipBytes() throws IOException { |
|
271 |
advance(Symbol.BYTES); |
|
272 |
if (in.getCurrentToken() == JsonToken.VALUE_STRING) { |
|
273 |
in.nextToken(); |
|
274 |
} else { |
|
275 |
throw error("bytes"); |
|
276 |
} |
|
277 |
} |
|
278 |
|
|
279 |
private void checkFixed(int size) throws IOException { |
|
280 |
advance(Symbol.FIXED); |
|
281 |
Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol(); |
|
282 |
if (size != top.size) { |
|
283 |
throw new AvroTypeException( |
|
284 |
"Incorrect length for fixed binary: expected " + |
|
285 |
top.size + " but received " + size + " bytes."); |
|
286 |
} |
|
287 |
} |
|
288 |
|
|
289 |
@Override |
|
290 |
public void readFixed(byte[] bytes, int start, int len) throws IOException { |
|
291 |
checkFixed(len); |
|
292 |
if (in.getCurrentToken() == JsonToken.VALUE_STRING) { |
|
293 |
byte[] result = readByteArray(); |
|
294 |
in.nextToken(); |
|
295 |
if (result.length != len) { |
|
296 |
throw new AvroTypeException("Expected fixed length " + len |
|
297 |
+ ", but got" + result.length); |
|
298 |
} |
|
299 |
System.arraycopy(result, 0, bytes, start, len); |
|
300 |
} else { |
|
301 |
throw error("fixed"); |
|
302 |
} |
|
303 |
} |
|
304 |
|
|
305 |
@Override |
|
306 |
public void skipFixed(int length) throws IOException { |
|
307 |
checkFixed(length); |
|
308 |
doSkipFixed(length); |
|
309 |
} |
|
310 |
|
|
311 |
private void doSkipFixed(int length) throws IOException { |
|
312 |
if (in.getCurrentToken() == JsonToken.VALUE_STRING) { |
|
313 |
byte[] result = readByteArray(); |
|
314 |
in.nextToken(); |
|
315 |
if (result.length != length) { |
|
316 |
throw new AvroTypeException("Expected fixed length " + length |
|
317 |
+ ", but got" + result.length); |
|
318 |
} |
|
319 |
} else { |
|
320 |
throw error("fixed"); |
|
321 |
} |
|
322 |
} |
|
323 |
|
|
324 |
@Override |
|
325 |
protected void skipFixed() throws IOException { |
|
326 |
advance(Symbol.FIXED); |
|
327 |
Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol(); |
|
328 |
doSkipFixed(top.size); |
|
329 |
} |
|
330 |
|
|
331 |
@Override |
|
332 |
public int readEnum() throws IOException { |
|
333 |
advance(Symbol.ENUM); |
|
334 |
Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol(); |
|
335 |
if (in.getCurrentToken() == JsonToken.VALUE_STRING) { |
|
336 |
in.getText(); |
|
337 |
int n = top.findLabel(in.getText()); |
|
338 |
if (n >= 0) { |
|
339 |
in.nextToken(); |
|
340 |
return n; |
|
341 |
} |
|
342 |
throw new AvroTypeException("Unknown symbol in enum " + in.getText()); |
|
343 |
} else { |
|
344 |
throw error("fixed"); |
|
345 |
} |
|
346 |
} |
|
347 |
|
|
348 |
@Override |
|
349 |
public long readArrayStart() throws IOException { |
|
350 |
advance(Symbol.ARRAY_START); |
|
351 |
if (in.getCurrentToken() == JsonToken.START_ARRAY) { |
|
352 |
in.nextToken(); |
|
353 |
return doArrayNext(); |
|
354 |
} else { |
|
355 |
throw error("array-start"); |
|
356 |
} |
|
357 |
} |
|
358 |
|
|
359 |
@Override |
|
360 |
public long arrayNext() throws IOException { |
|
361 |
advance(Symbol.ITEM_END); |
|
362 |
return doArrayNext(); |
|
363 |
} |
|
364 |
|
|
365 |
private long doArrayNext() throws IOException { |
|
366 |
if (in.getCurrentToken() == JsonToken.END_ARRAY) { |
|
367 |
parser.advance(Symbol.ARRAY_END); |
|
368 |
in.nextToken(); |
|
369 |
return 0; |
|
370 |
} else { |
|
371 |
return 1; |
|
372 |
} |
|
373 |
} |
|
374 |
|
|
375 |
@Override |
|
376 |
public long skipArray() throws IOException { |
|
377 |
advance(Symbol.ARRAY_START); |
|
378 |
if (in.getCurrentToken() == JsonToken.START_ARRAY) { |
|
379 |
in.skipChildren(); |
|
380 |
in.nextToken(); |
|
381 |
advance(Symbol.ARRAY_END); |
|
382 |
} else { |
|
383 |
throw error("array-start"); |
|
384 |
} |
|
385 |
return 0; |
|
386 |
} |
|
387 |
|
|
388 |
@Override |
|
389 |
public long readMapStart() throws IOException { |
|
390 |
advance(Symbol.MAP_START); |
|
391 |
if (in.getCurrentToken() == JsonToken.START_OBJECT) { |
|
392 |
in.nextToken(); |
|
393 |
return doMapNext(); |
|
394 |
} else { |
|
395 |
throw error("map-start"); |
|
396 |
} |
|
397 |
} |
|
398 |
|
|
399 |
@Override |
|
400 |
public long mapNext() throws IOException { |
|
401 |
advance(Symbol.ITEM_END); |
|
402 |
return doMapNext(); |
|
403 |
} |
|
404 |
|
|
405 |
private long doMapNext() throws IOException { |
|
406 |
if (in.getCurrentToken() == JsonToken.END_OBJECT) { |
|
407 |
in.nextToken(); |
|
408 |
advance(Symbol.MAP_END); |
|
409 |
return 0; |
|
410 |
} else { |
|
411 |
return 1; |
|
412 |
} |
|
413 |
} |
|
414 |
|
|
415 |
@Override |
|
416 |
public long skipMap() throws IOException { |
|
417 |
advance(Symbol.MAP_START); |
|
418 |
if (in.getCurrentToken() == JsonToken.START_OBJECT) { |
|
419 |
in.skipChildren(); |
|
420 |
in.nextToken(); |
|
421 |
advance(Symbol.MAP_END); |
|
422 |
} else { |
|
423 |
throw error("map-start"); |
|
424 |
} |
|
425 |
return 0; |
|
426 |
} |
|
427 |
|
|
428 |
@Override |
|
429 |
public int readIndex() throws IOException { |
|
430 |
advance(Symbol.UNION); |
|
431 |
Symbol.Alternative a = (Symbol.Alternative) parser.popSymbol(); |
|
432 |
|
|
433 |
String label; |
|
434 |
if (in.getCurrentToken() == JsonToken.VALUE_NULL) { |
|
435 |
label = "null"; |
|
436 |
} else if (isUnionOfNullWithSomething(a)){ |
|
437 |
int nullN = a.findLabel("null"); |
|
438 |
int otherN = (nullN+1)%2; |
|
439 |
// parser.pushSymbol(Symbol.UNION_END); |
|
440 |
parser.pushSymbol(a.getSymbol(otherN)); |
|
441 |
return otherN; |
|
442 |
} else if (in.getCurrentToken() == JsonToken.START_OBJECT && |
|
443 |
in.nextToken() == JsonToken.FIELD_NAME) { |
|
444 |
label = in.getText(); |
|
445 |
in.nextToken(); |
|
446 |
parser.pushSymbol(Symbol.UNION_END); |
|
447 |
} else { |
|
448 |
throw error("start-union"); |
|
449 |
} |
|
450 |
int n = a.findLabel(label); |
|
451 |
if (n < 0) |
|
452 |
throw new AvroTypeException("Unknown union branch " + label); |
|
453 |
parser.pushSymbol(a.getSymbol(n)); |
|
454 |
return n; |
|
455 |
} |
|
456 |
|
|
457 |
private static boolean isUnionOfNullWithSomething(Symbol.Alternative a){ |
|
458 |
return a.size()==2 && a.findLabel("null")!=-1; |
|
459 |
} |
|
460 |
|
|
461 |
@Override |
|
462 |
public Symbol doAction(Symbol input, Symbol top) throws IOException { |
|
463 |
if (top instanceof Symbol.FieldAdjustAction) { |
|
464 |
Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top; |
|
465 |
String name = fa.fname; |
|
466 |
if (currentReorderBuffer != null) { |
|
467 |
List<JsonElement> node = currentReorderBuffer.savedFields.get(name); |
|
468 |
if (node != null) { |
|
469 |
currentReorderBuffer.savedFields.remove(name); |
|
470 |
currentReorderBuffer.origParser = in; |
|
471 |
in = makeParser(node); |
|
472 |
return null; |
|
473 |
} |
|
474 |
} |
|
475 |
if (in.getCurrentToken() == JsonToken.FIELD_NAME) { |
|
476 |
do { |
|
477 |
String fn = in.getText(); |
|
478 |
in.nextToken(); |
|
479 |
if (name.equals(fn)) { |
|
480 |
return null; |
|
481 |
} else { |
|
482 |
if (currentReorderBuffer == null) { |
|
483 |
currentReorderBuffer = new ReorderBuffer(); |
|
484 |
} |
|
485 |
currentReorderBuffer.savedFields.put(fn, getVaueAsTree(in)); |
|
486 |
} |
|
487 |
} while (in.getCurrentToken() == JsonToken.FIELD_NAME); |
|
488 |
throw new AvroTypeException("Expected field name not found: " + fa.fname); |
|
489 |
} |
|
490 |
} else if (top == Symbol.FIELD_END) { |
|
491 |
if (currentReorderBuffer != null && currentReorderBuffer.origParser != null) { |
|
492 |
in = currentReorderBuffer.origParser; |
|
493 |
currentReorderBuffer.origParser = null; |
|
494 |
} |
|
495 |
} else if (top == Symbol.RECORD_START) { |
|
496 |
if (in.getCurrentToken() == JsonToken.START_OBJECT) { |
|
497 |
in.nextToken(); |
|
498 |
reorderBuffers.push(currentReorderBuffer); |
|
499 |
currentReorderBuffer = null; |
|
500 |
} else { |
|
501 |
throw error("record-start"); |
|
502 |
} |
|
503 |
} else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) { |
|
504 |
if (in.getCurrentToken() == JsonToken.END_OBJECT) { |
|
505 |
in.nextToken(); |
|
506 |
if (top == Symbol.RECORD_END) { |
|
507 |
if (currentReorderBuffer != null && !currentReorderBuffer.savedFields.isEmpty()) { |
|
508 |
throw error("Unknown fields: " + currentReorderBuffer.savedFields.keySet()); |
|
509 |
} |
|
510 |
currentReorderBuffer = reorderBuffers.pop(); |
|
511 |
} |
|
512 |
} else { |
|
513 |
throw error(top == Symbol.RECORD_END ? "record-end" : "union-end"); |
|
514 |
} |
|
515 |
} else { |
|
516 |
throw new AvroTypeException("Unknown action symbol " + top); |
|
517 |
} |
|
518 |
return null; |
|
519 |
} |
|
520 |
|
|
521 |
private static class JsonElement { |
|
522 |
public final JsonToken token; |
|
523 |
public final String value; |
|
524 |
public JsonElement(JsonToken t, String value) { |
|
525 |
this.token = t; |
|
526 |
this.value = value; |
|
527 |
} |
|
528 |
|
|
529 |
public JsonElement(JsonToken t) { |
|
530 |
this(t, null); |
|
531 |
} |
|
532 |
} |
|
533 |
|
|
534 |
private static List<JsonElement> getVaueAsTree(JsonParser in) throws IOException { |
|
535 |
int level = 0; |
|
536 |
List<JsonElement> result = new ArrayList<JsonElement>(); |
|
537 |
do { |
|
538 |
JsonToken t = in.getCurrentToken(); |
|
539 |
switch (t) { |
|
540 |
case START_OBJECT: |
|
541 |
case START_ARRAY: |
|
542 |
level++; |
|
543 |
result.add(new JsonElement(t)); |
|
544 |
break; |
|
545 |
case END_OBJECT: |
|
546 |
case END_ARRAY: |
|
547 |
level--; |
|
548 |
result.add(new JsonElement(t)); |
|
549 |
break; |
|
550 |
case FIELD_NAME: |
|
551 |
case VALUE_STRING: |
|
552 |
case VALUE_NUMBER_INT: |
|
553 |
case VALUE_NUMBER_FLOAT: |
|
554 |
case VALUE_TRUE: |
|
555 |
case VALUE_FALSE: |
|
556 |
case VALUE_NULL: |
|
557 |
result.add(new JsonElement(t, in.getText())); |
|
558 |
break; |
|
559 |
} |
|
560 |
in.nextToken(); |
|
561 |
} while (level != 0); |
|
562 |
result.add(new JsonElement(null)); |
|
563 |
return result; |
|
564 |
} |
|
565 |
|
|
566 |
private JsonParser makeParser(final List<JsonElement> elements) throws IOException { |
|
567 |
return new JsonParser() { |
|
568 |
int pos = 0; |
|
569 |
|
|
570 |
@Override |
|
571 |
public ObjectCodec getCodec() { |
|
572 |
throw new UnsupportedOperationException(); |
|
573 |
} |
|
574 |
|
|
575 |
@Override |
|
576 |
public void setCodec(ObjectCodec c) { |
|
577 |
throw new UnsupportedOperationException(); |
|
578 |
} |
|
579 |
|
|
580 |
@Override |
|
581 |
public void close() throws IOException { |
|
582 |
throw new UnsupportedOperationException(); |
|
583 |
} |
|
584 |
|
|
585 |
@Override |
|
586 |
public JsonToken nextToken() throws IOException { |
|
587 |
pos++; |
|
588 |
return elements.get(pos).token; |
|
589 |
} |
|
590 |
|
|
591 |
@Override |
|
592 |
public JsonParser skipChildren() throws IOException { |
|
593 |
int level = 0; |
|
594 |
do { |
|
595 |
switch(elements.get(pos++).token) { |
|
596 |
case START_ARRAY: |
|
597 |
case START_OBJECT: |
|
598 |
level++; |
|
599 |
break; |
|
600 |
case END_ARRAY: |
|
601 |
case END_OBJECT: |
|
602 |
level--; |
|
603 |
break; |
|
604 |
} |
|
605 |
} while (level > 0); |
|
606 |
return this; |
|
607 |
} |
|
608 |
|
|
609 |
@Override |
|
610 |
public boolean isClosed() { |
|
611 |
throw new UnsupportedOperationException(); |
|
612 |
} |
|
613 |
|
|
614 |
@Override |
|
615 |
public String getCurrentName() throws IOException { |
|
616 |
throw new UnsupportedOperationException(); |
|
617 |
} |
|
618 |
|
|
619 |
@Override |
|
620 |
public JsonStreamContext getParsingContext() { |
|
621 |
throw new UnsupportedOperationException(); |
|
622 |
} |
|
623 |
|
|
624 |
@Override |
|
625 |
public JsonLocation getTokenLocation() { |
|
626 |
throw new UnsupportedOperationException(); |
|
627 |
} |
|
628 |
|
|
629 |
@Override |
|
630 |
public JsonLocation getCurrentLocation() { |
|
631 |
throw new UnsupportedOperationException(); |
|
632 |
} |
|
633 |
|
|
634 |
@Override |
|
635 |
public String getText() throws IOException { |
|
636 |
return elements.get(pos).value; |
|
637 |
} |
|
638 |
|
|
639 |
@Override |
|
640 |
public char[] getTextCharacters() throws IOException { |
|
641 |
throw new UnsupportedOperationException(); |
|
642 |
} |
|
643 |
|
|
644 |
@Override |
|
645 |
public int getTextLength() throws IOException { |
|
646 |
throw new UnsupportedOperationException(); |
|
647 |
} |
|
648 |
|
|
649 |
@Override |
|
650 |
public int getTextOffset() throws IOException { |
|
651 |
throw new UnsupportedOperationException(); |
|
652 |
} |
|
653 |
|
|
654 |
@Override |
|
655 |
public Number getNumberValue() throws IOException { |
|
656 |
throw new UnsupportedOperationException(); |
|
657 |
} |
|
658 |
|
|
659 |
@Override |
|
660 |
public NumberType getNumberType() throws IOException { |
|
661 |
throw new UnsupportedOperationException(); |
|
662 |
} |
|
663 |
|
|
664 |
@Override |
|
665 |
public int getIntValue() throws IOException { |
|
666 |
return Integer.parseInt(getText()); |
|
667 |
} |
|
668 |
|
|
669 |
@Override |
|
670 |
public long getLongValue() throws IOException { |
|
671 |
return Long.parseLong(getText()); |
|
672 |
} |
|
673 |
|
|
674 |
@Override |
|
675 |
public BigInteger getBigIntegerValue() throws IOException { |
|
676 |
throw new UnsupportedOperationException(); |
|
677 |
} |
|
678 |
|
|
679 |
@Override |
|
680 |
public float getFloatValue() throws IOException { |
|
681 |
return Float.parseFloat(getText()); |
|
682 |
} |
|
683 |
|
|
684 |
@Override |
|
685 |
public double getDoubleValue() throws IOException { |
|
686 |
return Double.parseDouble(getText()); |
|
687 |
} |
|
688 |
|
|
689 |
@Override |
|
690 |
public BigDecimal getDecimalValue() throws IOException { |
|
691 |
throw new UnsupportedOperationException(); |
|
692 |
} |
|
693 |
|
|
694 |
@Override |
|
695 |
public byte[] getBinaryValue(Base64Variant b64variant) |
|
696 |
throws IOException { |
|
697 |
throw new UnsupportedOperationException(); |
|
698 |
} |
|
699 |
|
|
700 |
@Override |
|
701 |
public JsonToken getCurrentToken() { |
|
702 |
return elements.get(pos).token; |
|
703 |
} |
|
704 |
}; |
|
705 |
} |
|
706 |
|
|
707 |
private AvroTypeException error(String type) { |
|
708 |
return new AvroTypeException("Expected " + type + |
|
709 |
". Got " + in.getCurrentToken()); |
|
710 |
} |
|
711 |
|
|
712 |
} |
|
713 |
|
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/main/java/org/apache/avro/io/HackedJsonEncoder.java | ||
---|---|---|
1 |
/** |
|
2 |
* Licensed to the Apache Software Foundation (ASF) under one |
|
3 |
* or more contributor license agreements. See the NOTICE file |
|
4 |
* distributed with this work for additional information |
|
5 |
* regarding copyright ownership. The ASF licenses this file |
|
6 |
* to you under the Apache License, Version 2.0 (the |
|
7 |
* "License"); you may not use this file except in compliance |
|
8 |
* with the License. You may obtain a copy of the License at |
|
9 |
* |
|
10 |
* http://www.apache.org/licenses/LICENSE-2.0 |
|
11 |
* |
|
12 |
* Unless required by applicable law or agreed to in writing, software |
|
13 |
* distributed under the License is distributed on an "AS IS" BASIS, |
|
14 |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
15 |
* See the License for the specific language governing permissions and |
|
16 |
* limitations under the License. |
|
17 |
*/ |
|
18 |
package org.apache.avro.io; |
|
19 |
|
|
20 |
import java.io.IOException; |
|
21 |
import java.io.OutputStream; |
|
22 |
import java.nio.ByteBuffer; |
|
23 |
import java.util.BitSet; |
|
24 |
|
|
25 |
import org.apache.avro.AvroTypeException; |
|
26 |
import org.apache.avro.Schema; |
|
27 |
import org.apache.avro.io.parsing.JsonGrammarGenerator; |
|
28 |
import org.apache.avro.io.parsing.Parser; |
|
29 |
import org.apache.avro.io.parsing.Symbol; |
|
30 |
import org.apache.avro.util.Utf8; |
|
31 |
import org.codehaus.jackson.JsonEncoding; |
|
32 |
import org.codehaus.jackson.JsonFactory; |
|
33 |
import org.codehaus.jackson.JsonGenerator; |
|
34 |
import org.codehaus.jackson.util.MinimalPrettyPrinter; |
|
35 |
|
|
36 |
/** An {@link Encoder} for Avro's JSON data encoding. |
|
37 |
* </p> |
|
38 |
* Construct using {@link EncoderFactory}. |
|
39 |
* </p> |
|
40 |
* JsonEncoder buffers output, and data may not appear on the output |
|
41 |
* until {@link Encoder#flush()} is called. |
|
42 |
* </p> |
|
43 |
* JsonEncoder is not thread-safe. |
|
44 |
* */ |
|
45 |
public class HackedJsonEncoder extends ParsingEncoder implements Parser.ActionHandler { |
|
46 |
final Parser parser; |
|
47 |
private JsonGenerator out; |
|
48 |
/** |
|
49 |
* Has anything been written into the collections? |
|
50 |
*/ |
|
51 |
protected BitSet isEmpty = new BitSet(); |
|
52 |
|
|
53 |
public HackedJsonEncoder(Schema sc, OutputStream out) throws IOException { |
|
54 |
this(sc, getJsonGenerator(out)); |
|
55 |
} |
|
56 |
|
|
57 |
HackedJsonEncoder(Schema sc, JsonGenerator out) throws IOException { |
|
58 |
configure(out); |
|
59 |
this.parser = |
|
60 |
new Parser(new JsonGrammarGenerator().generate(sc), this); |
|
61 |
} |
|
62 |
|
|
63 |
@Override |
|
64 |
public void flush() throws IOException { |
|
65 |
parser.processImplicitActions(); |
|
66 |
if (out != null) { |
|
67 |
out.flush(); |
|
68 |
} |
|
69 |
} |
|
70 |
|
|
71 |
// by default, one object per line |
|
72 |
private static JsonGenerator getJsonGenerator(OutputStream out) |
|
73 |
throws IOException { |
|
74 |
if (null == out) |
|
75 |
throw new NullPointerException("OutputStream cannot be null"); |
|
76 |
JsonGenerator g |
|
77 |
= new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8); |
|
78 |
MinimalPrettyPrinter pp = new MinimalPrettyPrinter(); |
|
79 |
pp.setRootValueSeparator(System.getProperty("line.separator")); |
|
80 |
g.setPrettyPrinter(pp); |
|
81 |
return g; |
|
82 |
} |
|
83 |
|
|
84 |
/** |
|
85 |
* Reconfigures this JsonEncoder to use the output stream provided. |
|
86 |
* <p/> |
|
87 |
* If the OutputStream provided is null, a NullPointerException is thrown. |
|
88 |
* <p/> |
|
89 |
* Otherwise, this JsonEncoder will flush its current output and then |
|
90 |
* reconfigure its output to use a default UTF8 JsonGenerator that writes |
|
91 |
* to the provided OutputStream. |
|
92 |
* |
|
93 |
* @param out |
|
94 |
* The OutputStream to direct output to. Cannot be null. |
|
95 |
* @throws IOException |
|
96 |
* @return this JsonEncoder |
|
97 |
*/ |
|
98 |
public HackedJsonEncoder configure(OutputStream out) throws IOException { |
|
99 |
this.configure(getJsonGenerator(out)); |
|
100 |
return this; |
|
101 |
} |
|
102 |
|
|
103 |
/** |
|
104 |
* Reconfigures this JsonEncoder to output to the JsonGenerator provided. |
|
105 |
* <p/> |
|
106 |
* If the JsonGenerator provided is null, a NullPointerException is thrown. |
|
107 |
* <p/> |
|
108 |
* Otherwise, this JsonEncoder will flush its current output and then |
|
109 |
* reconfigure its output to use the provided JsonGenerator. |
|
110 |
* |
|
111 |
* @param generator |
|
112 |
* The JsonGenerator to direct output to. Cannot be null. |
|
113 |
* @throws IOException |
|
114 |
* @return this JsonEncoder |
|
115 |
*/ |
|
116 |
public HackedJsonEncoder configure(JsonGenerator generator) throws IOException { |
|
117 |
if (null == generator) |
|
118 |
throw new NullPointerException("JsonGenerator cannot be null"); |
|
119 |
if (null != parser) { |
|
120 |
flush(); |
|
121 |
} |
|
122 |
this.out = generator; |
|
123 |
return this; |
|
124 |
} |
|
125 |
|
|
126 |
@Override |
|
127 |
public void writeNull() throws IOException { |
|
128 |
parser.advance(Symbol.NULL); |
|
129 |
out.writeNull(); |
|
130 |
} |
|
131 |
|
|
132 |
@Override |
|
133 |
public void writeBoolean(boolean b) throws IOException { |
|
134 |
parser.advance(Symbol.BOOLEAN); |
|
135 |
out.writeBoolean(b); |
|
136 |
} |
|
137 |
|
|
138 |
@Override |
|
139 |
public void writeInt(int n) throws IOException { |
|
140 |
parser.advance(Symbol.INT); |
|
141 |
out.writeNumber(n); |
|
142 |
} |
|
143 |
|
|
144 |
@Override |
|
145 |
public void writeLong(long n) throws IOException { |
|
146 |
parser.advance(Symbol.LONG); |
|
147 |
out.writeNumber(n); |
|
148 |
} |
|
149 |
|
|
150 |
@Override |
|
151 |
public void writeFloat(float f) throws IOException { |
|
152 |
parser.advance(Symbol.FLOAT); |
|
153 |
out.writeNumber(f); |
|
154 |
} |
|
155 |
|
|
156 |
@Override |
|
157 |
public void writeDouble(double d) throws IOException { |
|
158 |
parser.advance(Symbol.DOUBLE); |
|
159 |
out.writeNumber(d); |
|
160 |
} |
|
161 |
|
|
162 |
@Override |
|
163 |
public void writeString(Utf8 utf8) throws IOException { |
|
164 |
writeString(utf8.toString()); |
|
165 |
} |
|
166 |
|
|
167 |
@Override |
|
168 |
public void writeString(String str) throws IOException { |
|
169 |
parser.advance(Symbol.STRING); |
|
170 |
if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) { |
|
171 |
parser.advance(Symbol.MAP_KEY_MARKER); |
|
172 |
out.writeFieldName(str); |
|
173 |
} else { |
|
174 |
out.writeString(str); |
|
175 |
} |
|
176 |
} |
|
177 |
|
|
178 |
@Override |
|
179 |
public void writeBytes(ByteBuffer bytes) throws IOException { |
|
180 |
if (bytes.hasArray()) { |
|
181 |
writeBytes(bytes.array(), bytes.position(), bytes.remaining()); |
|
182 |
} else { |
|
183 |
byte[] b = new byte[bytes.remaining()]; |
|
184 |
for (int i = 0; i < b.length; i++) { |
|
185 |
b[i] = bytes.get(); |
|
186 |
} |
|
187 |
writeBytes(b); |
|
188 |
} |
|
189 |
} |
|
190 |
|
|
191 |
@Override |
|
192 |
public void writeBytes(byte[] bytes, int start, int len) throws IOException { |
|
193 |
parser.advance(Symbol.BYTES); |
|
194 |
writeByteArray(bytes, start, len); |
|
195 |
} |
|
196 |
|
|
197 |
private void writeByteArray(byte[] bytes, int start, int len) |
|
198 |
throws IOException { |
|
199 |
out.writeString( |
|
200 |
new String(bytes, start, len, JsonDecoder.CHARSET)); |
|
201 |
} |
|
202 |
|
|
203 |
@Override |
|
204 |
public void writeFixed(byte[] bytes, int start, int len) throws IOException { |
|
205 |
parser.advance(Symbol.FIXED); |
|
206 |
Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol(); |
|
207 |
if (len != top.size) { |
|
208 |
throw new AvroTypeException( |
|
209 |
"Incorrect length for fixed binary: expected " + |
|
210 |
top.size + " but received " + len + " bytes."); |
|
211 |
} |
|
212 |
writeByteArray(bytes, start, len); |
|
213 |
} |
|
214 |
|
|
215 |
@Override |
|
216 |
public void writeEnum(int e) throws IOException { |
|
217 |
parser.advance(Symbol.ENUM); |
|
218 |
Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol(); |
|
219 |
if (e < 0 || e >= top.size) { |
|
220 |
throw new AvroTypeException( |
|
221 |
"Enumeration out of range: max is " + |
|
222 |
top.size + " but received " + e); |
|
223 |
} |
|
224 |
out.writeString(top.getLabel(e)); |
|
225 |
} |
|
226 |
|
|
227 |
@Override |
|
228 |
public void writeArrayStart() throws IOException { |
|
229 |
parser.advance(Symbol.ARRAY_START); |
|
230 |
out.writeStartArray(); |
|
231 |
push(); |
|
232 |
isEmpty.set(depth()); |
|
233 |
} |
|
234 |
|
|
235 |
@Override |
|
236 |
public void writeArrayEnd() throws IOException { |
|
237 |
if (! isEmpty.get(pos)) { |
|
238 |
parser.advance(Symbol.ITEM_END); |
|
239 |
} |
|
240 |
pop(); |
|
241 |
parser.advance(Symbol.ARRAY_END); |
|
242 |
out.writeEndArray(); |
|
243 |
} |
|
244 |
|
|
245 |
@Override |
|
246 |
public void writeMapStart() throws IOException { |
|
247 |
push(); |
|
248 |
isEmpty.set(depth()); |
|
249 |
|
|
250 |
parser.advance(Symbol.MAP_START); |
|
251 |
out.writeStartObject(); |
|
252 |
} |
|
253 |
|
|
254 |
@Override |
|
255 |
public void writeMapEnd() throws IOException { |
|
256 |
if (! isEmpty.get(pos)) { |
|
257 |
parser.advance(Symbol.ITEM_END); |
|
258 |
} |
|
259 |
pop(); |
|
260 |
|
|
261 |
parser.advance(Symbol.MAP_END); |
|
262 |
out.writeEndObject(); |
|
263 |
} |
|
264 |
|
|
265 |
@Override |
|
266 |
public void startItem() throws IOException { |
|
267 |
if (! isEmpty.get(pos)) { |
|
268 |
parser.advance(Symbol.ITEM_END); |
|
269 |
} |
|
270 |
super.startItem(); |
|
271 |
isEmpty.clear(depth()); |
|
272 |
} |
|
273 |
|
|
274 |
@Override |
|
275 |
public void writeIndex(int unionIndex) throws IOException { |
|
276 |
parser.advance(Symbol.UNION); |
|
277 |
Symbol.Alternative top = (Symbol.Alternative) parser.popSymbol(); |
|
278 |
Symbol symbol = top.getSymbol(unionIndex); |
|
279 |
if(symbol != Symbol.NULL && isUnionOfNullWithSomething(top)){ |
|
280 |
// parser.pushSymbol(Symbol.UNION_END); |
|
281 |
parser.pushSymbol(symbol); |
|
282 |
return; |
|
283 |
} |
|
284 |
if (symbol != Symbol.NULL) { |
|
285 |
out.writeStartObject(); |
|
286 |
out.writeFieldName(top.getLabel(unionIndex)); |
|
287 |
parser.pushSymbol(Symbol.UNION_END); |
|
288 |
} |
|
289 |
parser.pushSymbol(symbol); |
|
290 |
} |
|
291 |
|
|
292 |
private static boolean isUnionOfNullWithSomething(Symbol.Alternative a){ |
|
293 |
return a.size()==2 && a.findLabel("null")!=-1; |
|
294 |
} |
|
295 |
|
|
296 |
@Override |
|
297 |
public Symbol doAction(Symbol input, Symbol top) throws IOException { |
|
298 |
if (top instanceof Symbol.FieldAdjustAction) { |
|
299 |
Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top; |
|
300 |
out.writeFieldName(fa.fname); |
|
301 |
} else if (top == Symbol.RECORD_START) { |
|
302 |
out.writeStartObject(); |
|
303 |
} else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) { |
|
304 |
out.writeEndObject(); |
|
305 |
} else if (top != Symbol.FIELD_END) { |
|
306 |
throw new AvroTypeException("Unknown action symbol " + top); |
|
307 |
} |
|
308 |
return null; |
|
309 |
} |
|
310 |
} |
|
311 |
|
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/test/java/org/apache/avro/io/JsonCodersTest.java | ||
---|---|---|
1 |
package org.apache.avro.io; |
|
2 |
|
|
3 |
import static org.junit.Assert.assertEquals; |
|
4 |
import static org.junit.Assert.assertTrue; |
|
5 |
|
|
6 |
import java.io.BufferedReader; |
|
7 |
import java.io.ByteArrayOutputStream; |
|
8 |
import java.io.EOFException; |
|
9 |
import java.io.IOException; |
|
10 |
import java.io.InputStreamReader; |
|
11 |
import java.util.ArrayList; |
|
12 |
import java.util.List; |
|
13 |
|
|
14 |
import org.apache.avro.AvroTypeException; |
|
15 |
import org.apache.avro.Schema; |
|
16 |
import org.apache.avro.generic.GenericDatumReader; |
|
17 |
import org.apache.avro.generic.GenericDatumWriter; |
|
18 |
import org.apache.avro.generic.GenericRecord; |
|
19 |
import org.apache.avro.io.HackedJsonDecoder; |
|
20 |
import org.apache.avro.io.HackedJsonEncoder; |
|
21 |
import org.junit.Test; |
|
22 |
|
|
23 |
import org.apache.avro.io.schemas.Document; |
|
24 |
import org.apache.avro.io.schemas.PersonEntry; |
|
25 |
import org.apache.avro.io.schemas.PersonWithDocument; |
|
26 |
|
|
27 |
/** |
|
28 |
* @author Mateusz Kobos |
|
29 |
*/ |
|
30 |
public class JsonCodersTest { |
|
31 |
|
|
32 |
@Test |
|
33 |
public void testJsonSchemaSimpleWithNull() throws IOException { |
|
34 |
checkJsonReadWrite("{\"age\":10}\n{\"age\":null}", |
|
35 |
getCertainPersonNullableSchema()); |
|
36 |
} |
|
37 |
|
|
38 |
@Test |
|
39 |
public void testJsonSchemaWithoutNull() throws IOException { |
|
40 |
checkJsonReadWrite("{\"age\":10}\n{\"age\":33}", |
|
41 |
getCertainPersonSchema()); |
|
42 |
} |
|
43 |
|
|
44 |
@Test |
|
45 |
public void testJsonSchemaWritingNullToNonNullableField() |
|
46 |
throws IOException { |
|
47 |
checkJsonReadWrite("{\"age\":10}\n{\"age\":null}", |
|
48 |
getCertainPersonSchema(), true); |
|
49 |
} |
|
50 |
|
|
51 |
@Test |
|
52 |
public void testSimpleNullableField() throws IOException { |
|
53 |
checkJsonReadWrite( |
|
54 |
"{\"id\":1,\"title\":\"Interesting stuff\",\"authorIds\":[1,2,3]}", |
|
55 |
Document.SCHEMA$); |
|
56 |
checkJsonReadWrite("{\"id\":1,\"title\":null,\"authorIds\":[1,2,3]}", |
|
57 |
Document.SCHEMA$); |
|
58 |
// checkJsonReadWrite("{\"id\":1,\"authorIds\":[1,2,3]}", Document.SCHEMA$); |
|
59 |
checkJsonReadWrite("{\"id\":1,\"title\":\"Interesting stuff\"}", |
|
60 |
Document.SCHEMA$, true); |
|
61 |
} |
|
62 |
|
|
63 |
@Test |
|
64 |
public void testNestedStructures() throws IOException { |
|
65 |
String records = readFromResources( |
|
66 |
"org/apache/avro/io/nested_data.json"); |
|
67 |
checkJsonReadWrite(records, PersonWithDocument.SCHEMA$); |
|
68 |
} |
|
69 |
|
|
70 |
/** |
|
71 |
* When null is not one of the elements in the union, you have to specify |
|
72 |
* the type of the elements in the union explicitly |
|
73 |
*/ |
|
74 |
@Test |
|
75 |
public void testUnionWithoutNull() throws IOException { |
|
76 |
String records = readFromResources( |
|
77 |
"org/apache/avro/io/union_without_null.json"); |
|
78 |
checkJsonReadWrite(records, PersonEntry.SCHEMA$); |
|
79 |
checkJsonReadWrite("{\"id\":22,\"externalId\":3}", PersonEntry.SCHEMA$, |
|
80 |
true); |
|
81 |
} |
|
82 |
|
|
83 |
private static String readFromResources(String path) throws IOException { |
|
84 |
BufferedReader reader = new BufferedReader(new InputStreamReader(Thread |
|
85 |
.currentThread().getContextClassLoader() |
|
86 |
.getResourceAsStream(path))); |
|
87 |
StringBuffer buffer = new StringBuffer(); |
|
88 |
boolean isFirst = true; |
|
89 |
for (String line = reader.readLine(); line != null; line = reader |
|
90 |
.readLine()) { |
|
91 |
if (isFirst) { |
|
92 |
isFirst = false; |
|
93 |
} else { |
|
94 |
buffer.append("\n"); |
|
95 |
} |
|
96 |
isFirst = false; |
|
97 |
buffer.append(line); |
|
98 |
} |
|
99 |
return buffer.toString(); |
|
100 |
} |
|
101 |
|
|
102 |
private void checkJsonReadWrite(String jsonInput, Schema schema) |
|
103 |
throws IOException { |
|
104 |
checkJsonReadWrite(jsonInput, schema, false); |
|
105 |
} |
|
106 |
|
|
107 |
/** |
|
108 |
* Convert given JSON input string to Avro records, then convert it back to |
|
109 |
* JSON and compare it with the original. |
|
110 |
*/ |
|
111 |
private void checkJsonReadWrite(String jsonInput, Schema schema, |
|
112 |
boolean shouldThrowParsingException) throws IOException { |
|
113 |
try { |
|
114 |
List<GenericRecord> outRecords = toRecords(jsonInput, schema, |
|
115 |
schema); |
|
116 |
String jsonOutput = toJson(outRecords, schema); |
|
117 |
assertEquals(jsonInput, jsonOutput); |
|
118 |
} catch (AvroTypeException ex) { |
|
119 |
if (shouldThrowParsingException) { |
|
120 |
return; |
|
121 |
} else { |
|
122 |
throw new RuntimeException(ex); |
|
123 |
} |
|
124 |
} |
|
125 |
if (shouldThrowParsingException) { |
|
126 |
assertTrue( |
|
127 |
"This code should not have been reached because of previous " |
|
128 |
+ "exception thrown", false); |
|
129 |
} |
|
130 |
} |
|
131 |
|
|
132 |
private static Schema getCertainPersonSchema() throws IOException { |
|
133 |
return new Schema.Parser().parse(readFromResources( |
|
134 |
"org/apache/avro/io/schemas/certain_person.json")); |
|
135 |
} |
|
136 |
|
|
137 |
private static Schema getCertainPersonNullableSchema() throws IOException { |
|
138 |
return new Schema.Parser().parse(readFromResources( |
|
139 |
"org/apache/avro/io/schemas/certain_person_nullable.json")); |
|
140 |
} |
|
141 |
|
|
142 |
private static String toJson(List<GenericRecord> records, Schema schema) |
|
143 |
throws IOException { |
|
144 |
ByteArrayOutputStream output = new ByteArrayOutputStream(); |
|
145 |
HackedJsonEncoder jsonEncoder = new HackedJsonEncoder(schema, output); |
|
146 |
GenericDatumWriter<GenericRecord> writer = |
|
147 |
new GenericDatumWriter<GenericRecord>(schema); |
|
148 |
for (GenericRecord record : records) { |
|
149 |
writer.write(record, jsonEncoder); |
|
150 |
} |
|
151 |
jsonEncoder.flush(); |
|
152 |
output.flush(); |
|
153 |
return output.toString(); |
|
154 |
} |
|
155 |
|
|
156 |
private static List<GenericRecord> toRecords(String inputJson, |
|
157 |
Schema writerSchema, Schema readerSchema) throws IOException { |
|
158 |
HackedJsonDecoder jsonDecoder = new HackedJsonDecoder(writerSchema, |
|
159 |
inputJson); |
|
160 |
GenericDatumReader<GenericRecord> reader = |
|
161 |
new GenericDatumReader<GenericRecord>( |
|
162 |
writerSchema, readerSchema); |
|
163 |
List<GenericRecord> records = new ArrayList<GenericRecord>(); |
|
164 |
while (true) { |
|
165 |
try { |
|
166 |
GenericRecord record = reader.read(null, jsonDecoder); |
|
167 |
records.add(record); |
|
168 |
} catch (EOFException e) { |
|
169 |
break; |
|
170 |
} |
|
171 |
} |
|
172 |
return records; |
|
173 |
} |
|
174 |
} |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/test/resources/org/apache/avro/io/schemas/certain_person_nullable.json | ||
---|---|---|
1 |
{ |
|
2 |
"type":"record", |
|
3 |
"name":"CertainPerson", |
|
4 |
"fields": [{"name": "age", "type": ["null", "int"], "default": null}] |
|
5 |
} |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/test/resources/org/apache/avro/io/schemas/certain_person.json | ||
---|---|---|
1 |
{ |
|
2 |
"type":"record", |
|
3 |
"name":"CertainPerson", |
|
4 |
"fields": [{"name": "age", "type": "int"}] |
|
5 |
} |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/test/resources/org/apache/avro/io/schemas/nullable_structures.avdl | ||
---|---|---|
1 |
@namespace("org.apache.avro.io.schemas") |
|
2 |
protocol IIS{ |
|
3 |
record Document{ |
|
4 |
int id; |
|
5 |
union {null, string} title=null; |
|
6 |
array<int> authorIds; |
|
7 |
} |
|
8 |
|
|
9 |
record PersonWithDocument{ |
|
10 |
int id; |
|
11 |
union{null, string} name=null; |
|
12 |
union {null, Document} document; |
|
13 |
union {Document, null} documentWithNullLast; |
|
14 |
union {null, Document} documentWithNullDefault=null; |
|
15 |
union {null, array<Document>} documents; |
|
16 |
} |
|
17 |
|
|
18 |
record PersonEntry{ |
|
19 |
int id; |
|
20 |
union{int, string} externalId; |
|
21 |
} |
|
22 |
} |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/test/resources/org/apache/avro/io/nested_data.json | ||
---|---|---|
1 |
{"id":1,"name":null,"document":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documentWithNullLast":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documentWithNullDefault":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documents":[{"id":11,"title":"Some title11","authorIds":[1,3,5]},{"id":33,"title":"Some title33","authorIds":[2,4,6]}]} |
|
2 |
{"id":1,"name":"Some name","document":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documentWithNullLast":null,"documentWithNullDefault":null,"documents":null} |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/src/test/resources/org/apache/avro/io/union_without_null.json | ||
---|---|---|
1 |
{"id":22,"externalId":{"int":3}} |
|
2 |
{"id":11,"externalId":{"string":"some other string"}} |
modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3/README.markdown | ||
---|---|---|
1 |
This project contains patched version of |
|
2 |
|
|
3 |
**Avro library standard Avro-JSON conversion tools** |
|
4 |
|
|
5 |
See the `src/main/java/org/apache/avro/io/package-info.java` file for a detailed description of what the classes from this project do. |
Also available in: Unified diff
[maven-release-plugin] copy for tag icm-iis-3rdparty-avrojsoncoders-1.0.3