D-Net

package org.apache.avro.io;

import static org.junit.Assert.assertEquals;

import static org.junit.Assert.assertTrue;

import java.io.BufferedReader;

import java.io.ByteArrayOutputStream;

import java.io.EOFException;

import java.io.IOException;

import java.io.InputStreamReader;

import java.util.ArrayList;

import java.util.List;

import org.apache.avro.AvroTypeException;

import org.apache.avro.Schema;

import org.apache.avro.generic.GenericDatumReader;

import org.apache.avro.generic.GenericDatumWriter;

import org.apache.avro.generic.GenericRecord;

import org.apache.avro.io.HackedJsonDecoder;

import org.apache.avro.io.HackedJsonEncoder;

import org.junit.Test;

import org.apache.avro.io.schemas.Document;

import org.apache.avro.io.schemas.PersonEntry;

import org.apache.avro.io.schemas.PersonWithDocument;

/**

 * @author Mateusz Kobos

 */

public class JsonCodersTest {

	@Test

	public void testJsonSchemaSimpleWithNull() throws IOException {

		checkJsonReadWrite("{\"age\":10}\n{\"age\":null}",

				getCertainPersonNullableSchema());

	}

	@Test

	public void testJsonSchemaWithoutNull() throws IOException {

		checkJsonReadWrite("{\"age\":10}\n{\"age\":33}",

				getCertainPersonSchema());

	}

	@Test

	public void testJsonSchemaWritingNullToNonNullableField()

			throws IOException {

		checkJsonReadWrite("{\"age\":10}\n{\"age\":null}",

				getCertainPersonSchema(), true);

	}

	@Test

	public void testSimpleNullableField() throws IOException {

		checkJsonReadWrite(

				"{\"id\":1,\"title\":\"Interesting stuff\",\"authorIds\":[1,2,3]}",

				Document.SCHEMA$);

		checkJsonReadWrite("{\"id\":1,\"title\":null,\"authorIds\":[1,2,3]}",

				Document.SCHEMA$);

		// checkJsonReadWrite("{\"id\":1,\"authorIds\":[1,2,3]}", Document.SCHEMA$);

		checkJsonReadWrite("{\"id\":1,\"title\":\"Interesting stuff\"}",

				Document.SCHEMA$, true);

	}

	@Test

	public void testNestedStructures() throws IOException {

		String records = readFromResources(

				"org/apache/avro/io/nested_data.json");

		checkJsonReadWrite(records, PersonWithDocument.SCHEMA$);

	}

	/**

	 * When null is not one of the elements in the union, you have to specify

	 * the type of the elements in the union explicitly

	 */

	@Test

	public void testUnionWithoutNull() throws IOException {

		String records = readFromResources(

				"org/apache/avro/io/union_without_null.json");

		checkJsonReadWrite(records, PersonEntry.SCHEMA$);

		checkJsonReadWrite("{\"id\":22,\"externalId\":3}", PersonEntry.SCHEMA$,

				true);

	}

	private static String readFromResources(String path) throws IOException {

		BufferedReader reader = new BufferedReader(new InputStreamReader(Thread

				.currentThread().getContextClassLoader()

				.getResourceAsStream(path)));

		StringBuffer buffer = new StringBuffer();

		boolean isFirst = true;

		for (String line = reader.readLine(); line != null; line = reader

				.readLine()) {

			if (isFirst) {

				isFirst = false;

			} else {

				buffer.append("\n");

			}

			isFirst = false;

			buffer.append(line);

		}

		return buffer.toString();

	}

	private void checkJsonReadWrite(String jsonInput, Schema schema)

			throws IOException {

		checkJsonReadWrite(jsonInput, schema, false);

	}

	/**

	 * Convert given JSON input string to Avro records, then convert it back to

	 * JSON and compare it with the original.

	 */

	private void checkJsonReadWrite(String jsonInput, Schema schema,

			boolean shouldThrowParsingException) throws IOException {

		try {

			List<GenericRecord> outRecords = toRecords(jsonInput, schema,

					schema);

			String jsonOutput = toJson(outRecords, schema);

			assertEquals(jsonInput, jsonOutput);

		} catch (AvroTypeException ex) {

			if (shouldThrowParsingException) {

				return;

			} else {

				throw new RuntimeException(ex);

			}

		}

		if (shouldThrowParsingException) {

			assertTrue(

				"This code should not have been reached because of previous "

					+ "exception thrown", false);

		}

	}

	private static Schema getCertainPersonSchema() throws IOException {

		return new Schema.Parser().parse(readFromResources(

				"org/apache/avro/io/schemas/certain_person.json"));

	}

	private static Schema getCertainPersonNullableSchema() throws IOException {

		return new Schema.Parser().parse(readFromResources(

				"org/apache/avro/io/schemas/certain_person_nullable.json"));

	}

	private static String toJson(List<GenericRecord> records, Schema schema)

			throws IOException {

		ByteArrayOutputStream output = new ByteArrayOutputStream();

		HackedJsonEncoder jsonEncoder = new HackedJsonEncoder(schema, output);

		GenericDatumWriter<GenericRecord> writer = 

				new GenericDatumWriter<GenericRecord>(schema);

		for (GenericRecord record : records) {

			writer.write(record, jsonEncoder);

		}

		jsonEncoder.flush();

		output.flush();

		return output.toString();

	}

	private static List<GenericRecord> toRecords(String inputJson,

			Schema writerSchema, Schema readerSchema) throws IOException {

		HackedJsonDecoder jsonDecoder = new HackedJsonDecoder(writerSchema,

				inputJson);

		GenericDatumReader<GenericRecord> reader = 

				new GenericDatumReader<GenericRecord>(

						writerSchema, readerSchema);

		List<GenericRecord> records = new ArrayList<GenericRecord>();

		while (true) {

			try {

				GenericRecord record = reader.read(null, jsonDecoder);

				records.add(record);

			} catch (EOFException e) {

				break;

			}

		}

		return records;

	}

}

{"id":1,"name":null,"document":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documentWithNullLast":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documentWithNullDefault":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documents":[{"id":11,"title":"Some title11","authorIds":[1,3,5]},{"id":33,"title":"Some title33","authorIds":[2,4,6]}]}

{"id":1,"name":"Some name","document":{"id":33,"title":"Some title","authorIds":[1,3,5]},"documentWithNullLast":null,"documentWithNullDefault":null,"documents":null}

{

"type":"record",

"name":"CertainPerson",

"fields": [{"name": "age", "type": ["null", "int"], "default": null}]

}

{

"type":"record",

"name":"CertainPerson",

"fields": [{"name": "age", "type": "int"}]

}

@namespace("org.apache.avro.io.schemas")

protocol IIS{

   record Document{

      int id;

      union {null, string} title=null;

      array<int> authorIds;

   }

   record PersonWithDocument{

      int id;

      union{null, string} name=null;

      union {null, Document} document;

      union {Document, null} documentWithNullLast;

      union {null, Document} documentWithNullDefault=null;

      union {null, array<Document>} documents;

   }

   record PersonEntry{

      int id;

      union{int, string} externalId;

   }

}

{"id":22,"externalId":{"int":3}}

{"id":11,"externalId":{"string":"some other string"}}

/**

 * Licensed to the Apache Software Foundation (ASF) under one

 * or more contributor license agreements.  See the NOTICE file

 * distributed with this work for additional information

 * regarding copyright ownership.  The ASF licenses this file

 * to you under the Apache License, Version 2.0 (the

 * "License"); you may not use this file except in compliance

 * with the License.  You may obtain a copy of the License at

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package org.apache.avro.io;

import java.io.EOFException;

import java.io.IOException;

import java.io.InputStream;

import java.math.BigDecimal;

import java.math.BigInteger;

import java.nio.ByteBuffer;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Stack;

import org.apache.avro.AvroTypeException;

import org.apache.avro.Schema;

import org.apache.avro.io.parsing.JsonGrammarGenerator;

import org.apache.avro.io.parsing.Parser;

import org.apache.avro.io.parsing.Symbol;

import org.apache.avro.util.Utf8;

import org.codehaus.jackson.Base64Variant;

import org.codehaus.jackson.JsonFactory;

import org.codehaus.jackson.JsonLocation;

import org.codehaus.jackson.JsonParser;

import org.codehaus.jackson.JsonStreamContext;

import org.codehaus.jackson.JsonToken;

import org.codehaus.jackson.ObjectCodec;

/** A {@link Decoder} for Avro's JSON data encoding. 

 * </p>

 * Construct using {@link DecoderFactory}.

 * </p>

 * JsonDecoder is not thread-safe.

 * */

public class HackedJsonDecoder extends ParsingDecoder

  implements Parser.ActionHandler {

private JsonParser in;

  private static JsonFactory jsonFactory = new JsonFactory();

  Stack<ReorderBuffer> reorderBuffers = new Stack<ReorderBuffer>();

  ReorderBuffer currentReorderBuffer; 

  private static class ReorderBuffer {

    public Map<String, List<JsonElement>> savedFields = new HashMap<String, List<JsonElement>>();

    public JsonParser origParser = null; 

  }

  static final String CHARSET = "ISO-8859-1";

  private HackedJsonDecoder(Symbol root, InputStream in) throws IOException {

    super(root);

    configure(in);

  }

  private HackedJsonDecoder(Symbol root, String in) throws IOException {

    super(root);

    configure(in);

  }

  public HackedJsonDecoder(Schema schema, InputStream in) throws IOException {

    this(getSymbol(schema), in);

  }

  public HackedJsonDecoder(Schema schema, String in) throws IOException {

    this(getSymbol(schema), in);

  }

  private static Symbol getSymbol(Schema schema) {

    if (null == schema) {

      throw new NullPointerException("Schema cannot be null!");

    }

    return new JsonGrammarGenerator().generate(schema);

  }

  /**

   * Reconfigures this JsonDecoder to use the InputStream provided.

   * <p/>

   * If the InputStream provided is null, a NullPointerException is thrown.

   * <p/>

   * Otherwise, this JsonDecoder will reset its state and then

   * reconfigure its input.

   * @param in

   *   The IntputStream to read from. Cannot be null.

   * @throws IOException

   * @return this JsonDecoder

   */

  public HackedJsonDecoder configure(InputStream in) throws IOException {

    if (null == in) {

      throw new NullPointerException("InputStream to read from cannot be null!");

    }

    parser.reset();

    this.in = jsonFactory.createJsonParser(in);

    this.in.nextToken();

    return this;

  }

  /**

   * Reconfigures this JsonDecoder to use the String provided for input.

   * <p/>

   * If the String provided is null, a NullPointerException is thrown.

   * <p/>

   * Otherwise, this JsonDecoder will reset its state and then

   * reconfigure its input.

   * @param in

   *   The String to read from. Cannot be null.

   * @throws IOException

   * @return this JsonDecoder

   */

  public HackedJsonDecoder configure(String in) throws IOException {

    if (null == in) {

      throw new NullPointerException("String to read from cannot be null!");

    }

    parser.reset();

    this.in = new JsonFactory().createJsonParser(in);

    this.in.nextToken();

    return this;

  }

  private void advance(Symbol symbol) throws IOException {

    this.parser.processTrailingImplicitActions();

    if (in.getCurrentToken() == null && this.parser.depth() == 1)

      throw new EOFException();

    parser.advance(symbol);

  }

  @Override

  public void readNull() throws IOException {

    advance(Symbol.NULL);

    if (in.getCurrentToken() == JsonToken.VALUE_NULL) {

      in.nextToken();

    } else {

      throw error("null");

    }

  }

  @Override

  public boolean readBoolean() throws IOException {

    advance(Symbol.BOOLEAN);

    JsonToken t = in.getCurrentToken(); 

    if (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE) {

      in.nextToken();

      return t == JsonToken.VALUE_TRUE;

    } else {

      throw error("boolean");

    }

  }

  @Override

  public int readInt() throws IOException {

    advance(Symbol.INT);

    if (in.getCurrentToken().isNumeric()) {

      int result = in.getIntValue();

      in.nextToken();

      return result;

    } else {

      throw error("int");

    }

  }

  @Override

  public long readLong() throws IOException {

    advance(Symbol.LONG);

    if (in.getCurrentToken().isNumeric()) {

      long result = in.getLongValue();

      in.nextToken();

      return result;

    } else {

      throw error("long");

    }

  }

  @Override

  public float readFloat() throws IOException {

    advance(Symbol.FLOAT);

    if (in.getCurrentToken().isNumeric()) {

      float result = in.getFloatValue();

      in.nextToken();

      return result;

    } else {

      throw error("float");

    }

  }

  @Override

  public double readDouble() throws IOException {

    advance(Symbol.DOUBLE);

    if (in.getCurrentToken().isNumeric()) {

      double result = in.getDoubleValue();

      in.nextToken();

      return result;

    } else {

      throw error("double");

    }

  }

  @Override

  public Utf8 readString(Utf8 old) throws IOException {

    return new Utf8(readString());

  }

  @Override

  public String readString() throws IOException {

    advance(Symbol.STRING);

    if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) {

      parser.advance(Symbol.MAP_KEY_MARKER);

      if (in.getCurrentToken() != JsonToken.FIELD_NAME) {

        throw error("map-key");

      }

    } else {

      if (in.getCurrentToken() != JsonToken.VALUE_STRING) {

        throw error("string");

      }

    }

    String result = in.getText();

    in.nextToken();

    return result;

  }

  @Override

  public void skipString() throws IOException {

    advance(Symbol.STRING);

    if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) {

      parser.advance(Symbol.MAP_KEY_MARKER);

      if (in.getCurrentToken() != JsonToken.FIELD_NAME) {

        throw error("map-key");

      }

    } else {

      if (in.getCurrentToken() != JsonToken.VALUE_STRING) {

        throw error("string");

      }

    }

    in.nextToken();

  }

  @Override

  public ByteBuffer readBytes(ByteBuffer old) throws IOException {

    advance(Symbol.BYTES);

    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {

      byte[] result = readByteArray();

      in.nextToken();

      return ByteBuffer.wrap(result);

    } else {

      throw error("bytes");

    }

  }

  private byte[] readByteArray() throws IOException {

    byte[] result = in.getText().getBytes(CHARSET);

    return result;

  }

  @Override

  public void skipBytes() throws IOException {

    advance(Symbol.BYTES);

    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {

      in.nextToken();

    } else {

      throw error("bytes");

    }

  }

  private void checkFixed(int size) throws IOException {

    advance(Symbol.FIXED);

    Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol();

    if (size != top.size) {

      throw new AvroTypeException(

        "Incorrect length for fixed binary: expected " +

        top.size + " but received " + size + " bytes.");

    }

  }

  @Override

  public void readFixed(byte[] bytes, int start, int len) throws IOException {

    checkFixed(len);

    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {

      byte[] result = readByteArray();

      in.nextToken();

      if (result.length != len) {

        throw new AvroTypeException("Expected fixed length " + len

            + ", but got" + result.length);

      }

      System.arraycopy(result, 0, bytes, start, len);

    } else {

      throw error("fixed");

    }

  }

  @Override

  public void skipFixed(int length) throws IOException {

    checkFixed(length);

    doSkipFixed(length);

  }

  private void doSkipFixed(int length) throws IOException {

    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {

      byte[] result = readByteArray();

      in.nextToken();

      if (result.length != length) {

        throw new AvroTypeException("Expected fixed length " + length

            + ", but got" + result.length);

      }

    } else {

      throw error("fixed");

    }

  }

  @Override

  protected void skipFixed() throws IOException {

    advance(Symbol.FIXED);

    Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol();

    doSkipFixed(top.size);

  }

  @Override

  public int readEnum() throws IOException {

    advance(Symbol.ENUM);

    Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol();

    if (in.getCurrentToken() == JsonToken.VALUE_STRING) {

      in.getText();

      int n = top.findLabel(in.getText());

      if (n >= 0) {

        in.nextToken();

        return n;

      }

      throw new AvroTypeException("Unknown symbol in enum " + in.getText());

    } else {

      throw error("fixed");

    }

  }

  @Override

  public long readArrayStart() throws IOException {

    advance(Symbol.ARRAY_START);

    if (in.getCurrentToken() == JsonToken.START_ARRAY) {

      in.nextToken();

      return doArrayNext();

    } else {

      throw error("array-start");

    }

  }

  @Override

  public long arrayNext() throws IOException {

    advance(Symbol.ITEM_END);

    return doArrayNext();

  }

  private long doArrayNext() throws IOException {

    if (in.getCurrentToken() == JsonToken.END_ARRAY) {

      parser.advance(Symbol.ARRAY_END);

      in.nextToken();

      return 0;

    } else {

      return 1;

    }

  }

  @Override

  public long skipArray() throws IOException {

    advance(Symbol.ARRAY_START);

    if (in.getCurrentToken() == JsonToken.START_ARRAY) {

      in.skipChildren();

      in.nextToken();

      advance(Symbol.ARRAY_END);    

    } else {

      throw error("array-start");

    }

    return 0;

  }

  @Override

  public long readMapStart() throws IOException {

    advance(Symbol.MAP_START);

    if (in.getCurrentToken() == JsonToken.START_OBJECT) {

      in.nextToken();

      return doMapNext();

    } else {

      throw error("map-start");

    }

  }

  @Override

  public long mapNext() throws IOException {

    advance(Symbol.ITEM_END);

    return doMapNext();

  }

  private long doMapNext() throws IOException {

    if (in.getCurrentToken() == JsonToken.END_OBJECT) {

      in.nextToken();

      advance(Symbol.MAP_END);

      return 0;

    } else {

      return 1;

    }

  }

  @Override

  public long skipMap() throws IOException {

    advance(Symbol.MAP_START);

    if (in.getCurrentToken() == JsonToken.START_OBJECT) {

      in.skipChildren();

      in.nextToken();

      advance(Symbol.MAP_END);    

    } else {

      throw error("map-start");

    }

    return 0;

  }

  @Override

  public int readIndex() throws IOException {

    advance(Symbol.UNION);

    Symbol.Alternative a = (Symbol.Alternative) parser.popSymbol();

    String label;

    if (in.getCurrentToken() == JsonToken.VALUE_NULL) {

      label = "null";

    } else if (isUnionOfNullWithSomething(a)){

    	int nullN = a.findLabel("null");

    	int otherN = (nullN+1)%2;

//    	parser.pushSymbol(Symbol.UNION_END);

    	parser.pushSymbol(a.getSymbol(otherN));

    	return otherN;

    } else if (in.getCurrentToken() == JsonToken.START_OBJECT &&

               in.nextToken() == JsonToken.FIELD_NAME) {

      label = in.getText();

      in.nextToken();

      parser.pushSymbol(Symbol.UNION_END);

    } else {

      throw error("start-union");

    }

    int n = a.findLabel(label);

    if (n < 0)

      throw new AvroTypeException("Unknown union branch " + label);

    parser.pushSymbol(a.getSymbol(n));

    return n;

  }

  private static boolean isUnionOfNullWithSomething(Symbol.Alternative a){

	  return a.size()==2 && a.findLabel("null")!=-1;

  }

  @Override

  public Symbol doAction(Symbol input, Symbol top) throws IOException {

    if (top instanceof Symbol.FieldAdjustAction) {

        Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top;

        String name = fa.fname;

      if (currentReorderBuffer != null) {

        List<JsonElement> node = currentReorderBuffer.savedFields.get(name);

        if (node != null) {

          currentReorderBuffer.savedFields.remove(name);

          currentReorderBuffer.origParser = in;

          in = makeParser(node);

          return null;

        }

      }

      if (in.getCurrentToken() == JsonToken.FIELD_NAME) {

        do {

          String fn = in.getText();

          in.nextToken();

          if (name.equals(fn)) {

            return null;

          } else {

            if (currentReorderBuffer == null) {

              currentReorderBuffer = new ReorderBuffer();

            }

            currentReorderBuffer.savedFields.put(fn, getVaueAsTree(in));

          }

        } while (in.getCurrentToken() == JsonToken.FIELD_NAME);

        throw new AvroTypeException("Expected field name not found: " + fa.fname);

      }

    } else if (top == Symbol.FIELD_END) {

      if (currentReorderBuffer != null && currentReorderBuffer.origParser != null) {

        in = currentReorderBuffer.origParser;

        currentReorderBuffer.origParser = null;

      }

    } else if (top == Symbol.RECORD_START) {

      if (in.getCurrentToken() == JsonToken.START_OBJECT) {

        in.nextToken();

        reorderBuffers.push(currentReorderBuffer);

        currentReorderBuffer = null;

      } else {

        throw error("record-start");

      }

    } else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) {

      if (in.getCurrentToken() == JsonToken.END_OBJECT) {

        in.nextToken();

        if (top == Symbol.RECORD_END) {

          if (currentReorderBuffer != null && !currentReorderBuffer.savedFields.isEmpty()) {

            throw error("Unknown fields: " + currentReorderBuffer.savedFields.keySet());

          }

          currentReorderBuffer = reorderBuffers.pop();

        }

      } else {

        throw error(top == Symbol.RECORD_END ? "record-end" : "union-end");

      }

    } else {

      throw new AvroTypeException("Unknown action symbol " + top);

    }

    return null;

  }

  private static class JsonElement {

    public final JsonToken token;

    public final String value;

    public JsonElement(JsonToken t, String value) {

      this.token = t;

      this.value = value;

    }

    public JsonElement(JsonToken t) {

      this(t, null);

    }

  }

  private static List<JsonElement> getVaueAsTree(JsonParser in) throws IOException {

    int level = 0;

    List<JsonElement> result = new ArrayList<JsonElement>();

    do {

      JsonToken t = in.getCurrentToken();

      switch (t) {

      case START_OBJECT:

      case START_ARRAY:

        level++;

        result.add(new JsonElement(t));

        break;

      case END_OBJECT:

      case END_ARRAY:

        level--;

        result.add(new JsonElement(t));

        break;

      case FIELD_NAME:

      case VALUE_STRING:

      case VALUE_NUMBER_INT:

      case VALUE_NUMBER_FLOAT:

      case VALUE_TRUE:

      case VALUE_FALSE:

      case VALUE_NULL:

        result.add(new JsonElement(t, in.getText()));

        break;

      }

      in.nextToken();

    } while (level != 0);

    result.add(new JsonElement(null));

    return result;

  }

  private JsonParser makeParser(final List<JsonElement> elements) throws IOException {

    return new JsonParser() {

      int pos = 0;

      @Override

      public ObjectCodec getCodec() {

        throw new UnsupportedOperationException();

      }

      @Override

      public void setCodec(ObjectCodec c) {

        throw new UnsupportedOperationException();

      }

      @Override

      public void close() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public JsonToken nextToken() throws IOException {

        pos++;

        return elements.get(pos).token;

      }

      @Override

      public JsonParser skipChildren() throws IOException {

        int level = 0;

        do {

          switch(elements.get(pos++).token) {

          case START_ARRAY:

          case START_OBJECT:

            level++;

            break;

          case END_ARRAY:

          case END_OBJECT:

            level--;

            break;

          }

        } while (level > 0);

        return this;

      }

      @Override

      public boolean isClosed() {

        throw new UnsupportedOperationException();

      }

      @Override

      public String getCurrentName() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public JsonStreamContext getParsingContext() {

        throw new UnsupportedOperationException();

      }

      @Override

      public JsonLocation getTokenLocation() {

        throw new UnsupportedOperationException();

      }

      @Override

      public JsonLocation getCurrentLocation() {

        throw new UnsupportedOperationException();

      }

      @Override

      public String getText() throws IOException {

        return elements.get(pos).value;

      }

      @Override

      public char[] getTextCharacters() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public int getTextLength() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public int getTextOffset() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public Number getNumberValue() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public NumberType getNumberType() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public int getIntValue() throws IOException {

        return Integer.parseInt(getText());

      }

      @Override

      public long getLongValue() throws IOException {

        return Long.parseLong(getText());

      }

      @Override

      public BigInteger getBigIntegerValue() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public float getFloatValue() throws IOException {

        return Float.parseFloat(getText());

      }

      @Override

      public double getDoubleValue() throws IOException {

        return Double.parseDouble(getText());

      }

      @Override

      public BigDecimal getDecimalValue() throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public byte[] getBinaryValue(Base64Variant b64variant)

        throws IOException {

        throw new UnsupportedOperationException();

      }

      @Override

      public JsonToken getCurrentToken() {

        return elements.get(pos).token;

      }

    };

  }

  private AvroTypeException error(String type) {

    return new AvroTypeException("Expected " + type +

        ". Got " + in.getCurrentToken());

  }

}

/**

 * Provides hacked version of classes used to convert between Avro and JSON

 * formats. The hacked version allows for not defining explicitly types of

 * fields in a union when one of the union-ed fields is a null.

 * 

 * Generally the problem with the original implementation is in the way 

 * the Avro library converts JSON to Avro file when you use the schema with 

 * a "union" field (and we use this kind of field a lot to make a certain 

 * field null-able (e.g. "union {null, int} age")). 

 * 

 * For example when you use the schema (Avro IDL format):

 * 

 * record Document{

 *   int id;

 *   union {null, string} title=null;

 *   array<int> authorIds;

 * }

 * 

 * The JSON that Avro expects must have the type of the union field 

 * defined explicitly:

 * 

 * {"authorIds": [1, 20], "id": 20, "title": {"string": "An extraordinary book"}}

 * {"authorIds": [10, 6, 1], "id": 1, "title": {"string": "Basics of the basics"}}

 * {"authorIds": [], "id": 2, "title": null}

 * {"authorIds": [1, 6], "id": 6, "title": {"string": "Even more of interesting stuff"}}

 *

 * One way of dealing with it is just enforcing production of JSON conforming 

 * to this convention by our modules. However such an approach seems to 

 * introduce an unneeded dose of complexity to the produced JSON, 

 * so I decided to modify the Avro library to make it accept JSON without the

 * type in the union field defined explicitly. 

 *

 * This convention of defining JSON structure is necessary in the general case 

 * (see http://mail-archives.apache.org/mod_mbox/avro-user/201206.mbox/%3CCALEq1Z84=20PfJahT7DKtXEAGPFfOoHtVYXiyi+O+cZYGdXBNQ@mail.gmail.com%3E), 

 * but in our case of using "union" only to mark a field nullable 

 * it does not seem to be needed.

 */

package org.apache.avro.io;

/**

 * Licensed to the Apache Software Foundation (ASF) under one

 * or more contributor license agreements.  See the NOTICE file

 * distributed with this work for additional information

 * regarding copyright ownership.  The ASF licenses this file

 * to you under the Apache License, Version 2.0 (the

 * "License"); you may not use this file except in compliance

 * with the License.  You may obtain a copy of the License at

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package org.apache.avro.io;

import java.io.IOException;

import java.io.OutputStream;

import java.nio.ByteBuffer;

import java.util.BitSet;

import org.apache.avro.AvroTypeException;

import org.apache.avro.Schema;

import org.apache.avro.io.parsing.JsonGrammarGenerator;

import org.apache.avro.io.parsing.Parser;

import org.apache.avro.io.parsing.Symbol;

import org.apache.avro.util.Utf8;

import org.codehaus.jackson.JsonEncoding;

import org.codehaus.jackson.JsonFactory;

import org.codehaus.jackson.JsonGenerator;

import org.codehaus.jackson.util.MinimalPrettyPrinter;

/** An {@link Encoder} for Avro's JSON data encoding. 

 * </p>

 * Construct using {@link EncoderFactory}.

 * </p>

 * JsonEncoder buffers output, and data may not appear on the output

 * until {@link Encoder#flush()} is called.

 * </p>

 * JsonEncoder is not thread-safe.

 * */

public class HackedJsonEncoder extends ParsingEncoder implements Parser.ActionHandler {

  final Parser parser;

  private JsonGenerator out;

  /**

   * Has anything been written into the collections?

   */

  protected BitSet isEmpty = new BitSet();

  public HackedJsonEncoder(Schema sc, OutputStream out) throws IOException {

    this(sc, getJsonGenerator(out));

  }

  HackedJsonEncoder(Schema sc, JsonGenerator out) throws IOException {

    configure(out);

    this.parser =

      new Parser(new JsonGrammarGenerator().generate(sc), this);

  }

  @Override

  public void flush() throws IOException {

    parser.processImplicitActions();

    if (out != null) {

      out.flush();

    }

  }

  // by default, one object per line

  private static JsonGenerator getJsonGenerator(OutputStream out)

      throws IOException {

    if (null == out)

      throw new NullPointerException("OutputStream cannot be null"); 

    JsonGenerator g

      = new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);

    MinimalPrettyPrinter pp = new MinimalPrettyPrinter();

    pp.setRootValueSeparator(System.getProperty("line.separator"));

    g.setPrettyPrinter(pp);

    return g;

  }

  /**

   * Reconfigures this JsonEncoder to use the output stream provided.

   * <p/>

   * If the OutputStream provided is null, a NullPointerException is thrown.

   * <p/>

   * Otherwise, this JsonEncoder will flush its current output and then

   * reconfigure its output to use a default UTF8 JsonGenerator that writes

   * to the provided OutputStream.

   * 

   * @param out

   *          The OutputStream to direct output to. Cannot be null.

   * @throws IOException

   * @return this JsonEncoder

   */

  public HackedJsonEncoder configure(OutputStream out) throws IOException {

    this.configure(getJsonGenerator(out));

    return this;

  }

  /**

   * Reconfigures this JsonEncoder to output to the JsonGenerator provided.

   * <p/>

   * If the JsonGenerator provided is null, a NullPointerException is thrown.

   * <p/>

   * Otherwise, this JsonEncoder will flush its current output and then

   * reconfigure its output to use the provided JsonGenerator.

   * 

   * @param generator

   *          The JsonGenerator to direct output to. Cannot be null.

   * @throws IOException

   * @return this JsonEncoder

   */

  public HackedJsonEncoder configure(JsonGenerator generator) throws IOException {

    if (null == generator)

      throw new NullPointerException("JsonGenerator cannot be null");

    if (null != parser) {

      flush();

    }

    this.out = generator;

    return this;

  }

  @Override

  public void writeNull() throws IOException {

    parser.advance(Symbol.NULL);

    out.writeNull();

  }

  @Override

  public void writeBoolean(boolean b) throws IOException {

    parser.advance(Symbol.BOOLEAN);

    out.writeBoolean(b);

  }

  @Override

  public void writeInt(int n) throws IOException {

    parser.advance(Symbol.INT);

    out.writeNumber(n);

  }

  @Override

  public void writeLong(long n) throws IOException {

    parser.advance(Symbol.LONG);

    out.writeNumber(n);

  }

  @Override

  public void writeFloat(float f) throws IOException {

    parser.advance(Symbol.FLOAT);

    out.writeNumber(f);

  }

  @Override

  public void writeDouble(double d) throws IOException {

    parser.advance(Symbol.DOUBLE);

    out.writeNumber(d);

  }

  @Override

  public void writeString(Utf8 utf8) throws IOException {

    writeString(utf8.toString());

  }

  @Override 

  public void writeString(String str) throws IOException {

    parser.advance(Symbol.STRING);

    if (parser.topSymbol() == Symbol.MAP_KEY_MARKER) {

      parser.advance(Symbol.MAP_KEY_MARKER);

      out.writeFieldName(str);

    } else {

      out.writeString(str);

    }

  }

  @Override

  public void writeBytes(ByteBuffer bytes) throws IOException {

    if (bytes.hasArray()) {

      writeBytes(bytes.array(), bytes.position(), bytes.remaining());

    } else {

      byte[] b = new byte[bytes.remaining()];

      for (int i = 0; i < b.length; i++) {

        b[i] = bytes.get();

      }

      writeBytes(b);

    }

  }

  @Override

  public void writeBytes(byte[] bytes, int start, int len) throws IOException {

    parser.advance(Symbol.BYTES);

    writeByteArray(bytes, start, len);

  }

  private void writeByteArray(byte[] bytes, int start, int len)

    throws IOException {

    out.writeString(

        new String(bytes, start, len, JsonDecoder.CHARSET));

  }

  @Override

  public void writeFixed(byte[] bytes, int start, int len) throws IOException {

    parser.advance(Symbol.FIXED);

    Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol();

    if (len != top.size) {

      throw new AvroTypeException(

        "Incorrect length for fixed binary: expected " +

        top.size + " but received " + len + " bytes.");

    }

    writeByteArray(bytes, start, len);

  }

  @Override

  public void writeEnum(int e) throws IOException {

    parser.advance(Symbol.ENUM);

    Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction) parser.popSymbol();

    if (e < 0 || e >= top.size) {

      throw new AvroTypeException(

          "Enumeration out of range: max is " +

          top.size + " but received " + e);

    }

    out.writeString(top.getLabel(e));

  }

  @Override

  public void writeArrayStart() throws IOException {

    parser.advance(Symbol.ARRAY_START);

    out.writeStartArray();

    push();

    isEmpty.set(depth());

  }

  @Override

  public void writeArrayEnd() throws IOException {

    if (! isEmpty.get(pos)) {

      parser.advance(Symbol.ITEM_END);

    }

    pop();

    parser.advance(Symbol.ARRAY_END);

    out.writeEndArray();

  }

  @Override

  public void writeMapStart() throws IOException {

    push();

    isEmpty.set(depth());

    parser.advance(Symbol.MAP_START);

    out.writeStartObject();

  }

  @Override

  public void writeMapEnd() throws IOException {

    if (! isEmpty.get(pos)) {

      parser.advance(Symbol.ITEM_END);

    }

    pop();

    parser.advance(Symbol.MAP_END);

    out.writeEndObject();

  }

  @Override

  public void startItem() throws IOException {

    if (! isEmpty.get(pos)) {

      parser.advance(Symbol.ITEM_END);

    }

    super.startItem();

    isEmpty.clear(depth());

  }

  @Override

  public void writeIndex(int unionIndex) throws IOException {

    parser.advance(Symbol.UNION);

    Symbol.Alternative top = (Symbol.Alternative) parser.popSymbol();

    Symbol symbol = top.getSymbol(unionIndex);

    if(symbol != Symbol.NULL && isUnionOfNullWithSomething(top)){

//    	parser.pushSymbol(Symbol.UNION_END);

    	parser.pushSymbol(symbol);

    	return;

    }

    if (symbol != Symbol.NULL) {

      out.writeStartObject();

      out.writeFieldName(top.getLabel(unionIndex));

      parser.pushSymbol(Symbol.UNION_END);

    }

    parser.pushSymbol(symbol);

  }

  private static boolean isUnionOfNullWithSomething(Symbol.Alternative a){

	  return a.size()==2 && a.findLabel("null")!=-1;

  }

  @Override

  public Symbol doAction(Symbol input, Symbol top) throws IOException {

    if (top instanceof Symbol.FieldAdjustAction) {

      Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction) top;

      out.writeFieldName(fa.fname);

    } else if (top == Symbol.RECORD_START) {

      out.writeStartObject();

    } else if (top == Symbol.RECORD_END || top == Symbol.UNION_END) {

      out.writeEndObject();

    } else if (top != Symbol.FIELD_END) {

      throw new AvroTypeException("Unknown action symbol " + top);

    }

    return null;

  }

}

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

	<parent>

    		<groupId>eu.dnetlib</groupId>

	        <artifactId>dnet45-parent</artifactId>

            <version>1.0.0</version>

	</parent>

	<modelVersion>4.0.0</modelVersion>

	<artifactId>icm-iis-3rdparty-avrojsoncoders</artifactId>

	<packaging>jar</packaging>

	<version>1.0.3</version>

	<scm>

	    <developerConnection>

    		scm:svn:https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/icm-iis-3rdparty-avrojsoncoders/tags/icm-iis-3rdparty-avrojsoncoders-1.0.3

	    </developerConnection>

	 </scm>

	<properties>

		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

	</properties>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>4.10</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.apache.avro</groupId>

			<artifactId>avro</artifactId>

			<version>${iis.avro.version}</version>

		</dependency>

	</dependencies>

	<build>

		<plugins>

			<!-- Plugin that generates Java classes from Avro schemas -->

			<plugin>

				<groupId>org.apache.avro</groupId>

				<artifactId>avro-maven-plugin</artifactId>

				<version>${iis.avro.version}</version>

				<executions>

					<execution>

						<phase>generate-sources</phase>

						<goals>

							<goal>schema</goal>

							<goal>idl-protocol</goal>

						</goals>

						<configuration>

							<sourceDirectory>${project.basedir}/src/test/resources/</sourceDirectory>

							<outputDirectory>${project.basedir}/target/generated-sources/java/</outputDirectory>

						</configuration>

					</execution>

				</executions>

			</plugin>

			<!-- This plugin makes the Maven->Update Project Configuration not forget 

				about the "target/generated-sources/java" source path -->

			<plugin>

				<groupId>org.codehaus.mojo</groupId>

				<artifactId>build-helper-maven-plugin</artifactId>

				<executions>

					<execution>

						<id>add-source</id>

						<phase>generate-sources</phase>

						<goals>

							<goal>add-source</goal>

						</goals>

						<configuration>

							<sources>

								<source>${project.build.directory}/generated-sources/java/</source>

							</sources>

						</configuration>

					</execution>

				</executions>

			</plugin>

		</plugins>

		<pluginManagement>

			<plugins>

				<!--This plugin's configuration is used to store Eclipse m2e settings 

					only. It has no influence on the Maven build itself. -->

				<plugin>

					<groupId>org.eclipse.m2e</groupId>

					<artifactId>lifecycle-mapping</artifactId>

					<version>1.0.0</version>

					<configuration>

						<lifecycleMappingMetadata>

							<pluginExecutions>

								<pluginExecution>

									<pluginExecutionFilter>

										<groupId>

											org.apache.avro

										</groupId>

										<artifactId>

											avro-maven-plugin

										</artifactId>

										<versionRange>

											[1.7.4,)

										</versionRange>

										<goals>

											<goal>idl-protocol</goal>

											<goal>schema</goal>

										</goals>

									</pluginExecutionFilter>

									<action>

										<ignore />

									</action>

								</pluginExecution>

								<pluginExecution>

									<pluginExecutionFilter>

										<groupId>

											org.codehaus.mojo

										</groupId>

										<artifactId>

											build-helper-maven-plugin

										</artifactId>

										<versionRange>

											[1.7,)

										</versionRange>

										<goals>

											<goal>add-source</goal>

										</goals>

									</pluginExecutionFilter>

									<action>

										<ignore />

									</action>

								</pluginExecution>

							</pluginExecutions>

						</lifecycleMappingMetadata>

					</configuration>

				</plugin>

			</plugins>

		</pluginManagement>

	</build>

	<repositories>

	</repositories>

</project>

This project contains patched version of 

**Avro library standard Avro-JSON conversion tools**

See the `src/main/java/org/apache/avro/io/package-info.java` file for a detailed description of what the classes from this project do.